{% import 'templates/_macros.j2' as elk_macros %} ########################## Metricbeat Configuration ########################### # This file is a full configuration example documenting all non-deprecated # options in comments. For a shorter configuration example, that contains only # the most common options, please see metricbeat.yml in the same directory. # # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/metricbeat/index.html #============================ Config Reloading =============================== # Config reloading allows to dynamically load modules. Each file which is # monitored must contain one or multiple modules as a list. metricbeat.config.modules: # Glob pattern for configuration reloading path: ${path.config}/conf.d/*.yml # Period on which files under path should be checked for changes reload.period: 30s # Set to true to enable config reloading reload.enabled: false # Maximum amount of time to randomly delay the start of a metricset. Use 0 to # disable startup delay. metricbeat.max_start_delay: 10s #============================== Autodiscover =================================== # Autodiscover allows you to detect changes in the system and spawn new modules # as they happen. #metricbeat.autodiscover: # List of enabled autodiscover providers # providers: # - type: docker # templates: # - condition: # equals.docker.container.image: etcd # config: # - module: etcd # metricsets: ["leader", "self", "store"] # period: 30s # hosts: ["${host}:2379"] #========================== Modules configuration ============================ {% set metric_sets = ['network', 'process', 'process_summary', 'uptime'] %} {% if physical_host is defined and physical_host != inventory_hostname %} {% set host_mount_devices = (hostvars[physical_host]['ansible_mounts'] | map(attribute='device') | list) %} {% set container_mount_devices = (ansible_mounts | map(attribute='device') | list) %} {% if (((container_mount_devices | difference(host_mount_devices)) | length) > 0) | bool %} {% set _ = metric_sets.extend(['filesystem', 'fsstat']) %} {% endif %} {% else %} {% set _ = metric_sets.extend(['cpu', 'load', 'memory', 'core', 'diskio', 'raid', 'socket', 'filesystem', 'fsstat']) %} {% endif %} metricbeat.modules: #------------------------------- System Module ------------------------------- - module: system metricsets: {{ metric_sets }} #- cpu # CPU usage #- load # CPU load averages #- memory # Memory usage #- network # Network IO #- process # Per process metrics #- process_summary # Process summary #- uptime # System Uptime #- socket_summary # Socket summary #- core # Per CPU core usage #- diskio # Disk IO #- filesystem # File system usage for each mountpoint #- fsstat # File system summary metrics #- raid # Raid #- socket # Sockets and connection info (linux only) enabled: true period: 60s processes: ['.*'] # Configure the metric types that are included by these metricsets. cpu.metrics: ["percentages"] # The other available options are normalized_percentages and ticks. core.metrics: ["percentages"] # The other available option is ticks. # A list of filesystem types to ignore. The filesystem metricset will not # collect data from filesystems matching any of the specified types, and # fsstats will not include data from these filesystems in its summary stats. # If not set, types associated to virtual filesystems are automatically # added when this information is available in the system (e.g. the list of # `nodev` types in `/proc/filesystem`). #filesystem.ignore_types: [] # These options allow you to filter out all processes that are not # in the top N by CPU or memory, in order to reduce the number of documents created. # If both the `by_cpu` and `by_memory` options are used, the union of the two sets # is included. process.include_top_n: # Set to false to disable this feature and include all processes enabled: true # How many processes to include from the top by CPU. The processes are sorted # by the `system.process.cpu.total.pct` field. by_cpu: 20 # How many processes to include from the top by memory. The processes are sorted # by the `system.process.memory.rss.bytes` field. by_memory: 20 # If false, cmdline of a process is not cached. process.cmdline.cache.enabled: true # Enable collection of cgroup metrics from processes on Linux. process.cgroups.enabled: {{ ansible_service_mgr == 'systemd' }} # A list of regular expressions used to whitelist environment variables # reported with the process metricset's events. Defaults to empty. #process.env.whitelist: [] # Include the cumulative CPU tick values with the process metrics. Defaults # to false. #process.include_cpu_ticks: false # Raid mount point to monitor #raid.mount_point: '/' # Configure reverse DNS lookup on remote IP addresses in the socket metricset. socket.reverse_lookup.enabled: true socket.reverse_lookup.success_ttl: 60s socket.reverse_lookup.failure_ttl: 60s # Diskio configurations #diskio.include_devices: [] #------------------------------ Aerospike Module ----------------------------- #- module: aerospike # metricsets: ["namespace"] # enabled: true # period: 10s # hosts: ["localhost:3000"] # #------------------------------- Apache Module ------------------------------- {% if apache_enabled | default(false) | bool %} - module: apache metricsets: ["status"] period: 30s enabled: true # Apache hosts hosts: ["http://127.0.1.1:18181"] # Path to server status. Default server-status #server_status_path: "server-status" # Username of hosts. Empty by default #username: username # Password of hosts. Empty by default #password: password {% endif %} #-------------------------------- Ceph Module -------------------------------- {% if ceph_restapi_enabled | default(false) | bool %} - module: ceph metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk", "osd_tree"] period: 30s hosts: {{ ceph_stats_hosts | to_json }} enabled: true {% endif %} #------------------------------ Couchbase Module ----------------------------- #- module: couchbase # metricsets: ["bucket", "cluster", "node"] # period: 10s # hosts: ["localhost:8091"] # enabled: true # #------------------------------- couchdb Module ------------------------------ {% if couchdb_enabled | bool %} - module: couchdb metricsets: ["server"] period: 10s hosts: ["localhost:5984"] {% endif %} #------------------------------- Docker Module ------------------------------- {% if docker_enabled | default(false) | bool %} - module: docker metricsets: - "container" - "cpu" - "diskio" - "event" - "healthcheck" - "info" #- "image" - "memory" - "network" hosts: ["unix:///var/run/docker.sock"] period: 30s enabled: true # If set to true, replace dots in labels with `_`. #labels.dedot: false # If set to true, collects metrics per core. #cpu.cores: true # To connect to Docker over TLS you must specify a client and CA certificate. #ssl: #certificate_authority: "/etc/pki/root/ca.pem" #certificate: "/etc/pki/client/cert.pem" #key: "/etc/pki/client/cert.key" {% endif %} #----------------------------- Dropwizard Module ----------------------------- #- module: dropwizard # metricsets: ["collector"] # period: 10s # hosts: ["localhost:8080"] # metrics_path: /metrics/metrics # namespace: example # enabled: true # #---------------------------- Elasticsearch Module --------------------------- {% if inventory_hostname in (groups['elastic-logstash'] | union(groups['kibana'])) %} - module: elasticsearch metricsets: - node - node_stats - index - index_recovery - index_summary - shard - ml_job enabled: true period: 30s hosts: ["localhost:{{ elastic_port }}"] #username: "elastic" #password: "changeme" #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # Set to false to fetch all entries #index_recovery.active_only: true # Set to true to send data collected by module to X-Pack # Monitoring instead of metricbeat-* indices. #xpack.enabled: false {% endif %} #----------------------------- envoyproxy Module ----------------------------- {% if envoyproxy_enabled | bool %} - module: envoyproxy metricsets: ["server"] period: 10s hosts: ["localhost:9901"] {% endif %} #-------------------------------- Etcd Module -------------------------------- {% if etcd_enabled | default(false) | bool %} - module: etcd metricsets: ["leader", "self", "store"] enabled: true period: 30s hosts: ["localhost:2379"] {% endif %} #------------------------------- Golang Module ------------------------------- #- module: golang #metricsets: # - expvar # - heap # period: 10s # hosts: ["localhost:6060"] # heap.path: "/debug/vars" # expvar: # namespace: "example" # path: "/debug/vars" # #------------------------------ Graphite Module ------------------------------ #- module: graphite # metricsets: ["server"] # enabled: true # Host address to listen on. Default localhost. #host: localhost # Listening port. Default 2003. #port: 2003 # Protocol to listen on. This can be udp or tcp. Default udp. #protocol: "udp" # Receive buffer size in bytes #receive_buffer_size: 1024 #templates: # - filter: "test.*.bash.*" # This would match metrics like test.localhost.bash.stats # namespace: "test" # template: ".host.shell.metric*" # test.localhost.bash.stats would become metric=stats and tags host=localhost,shell=bash # delimiter: "_" #------------------------------- HAProxy Module ------------------------------ {% if haproxy_enabled | default(false) | bool %} - module: haproxy metricsets: ["info", "stat"] period: 30s hosts: [ {{ elastic_metricbeat_haproxy_monitoring_hosts }} ] enabled: true {% endif %} #-------------------------------- HTTP Module -------------------------------- #- module: http #metricsets: # - json #period: 10s #hosts: ["localhost:80"] #namespace: "json_namespace" #path: "/" #body: "" #method: "GET" #username: "user" #password: "secret" #request.enabled: false #response.enabled: false #json.is_array: false #dedot.enabled: false - module: http #metricsets: # - server host: "localhost" port: "8080" enabled: false #paths: # - path: "/foo" # namespace: "foo" # fields: # added to the the response in root. overwrites existing fields # key: "value" #------------------------------- Jolokia Module ------------------------------ #- module: jolokia #metricsets: ["jmx"] #period: 10s #hosts: ["localhost"] #namespace: "metrics" #path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" #username: "user" #password: "secret" #jmx.mappings: #- mbean: 'java.lang:type=Runtime' # attributes: # - attr: Uptime # field: uptime #- mbean: 'java.lang:type=Memory' # attributes: # - attr: HeapMemoryUsage # field: memory.heap_usage # - attr: NonHeapMemoryUsage # field: memory.non_heap_usage # GC Metrics - this depends on what is available on your JVM #- mbean: 'java.lang:type=GarbageCollector,name=ConcurrentMarkSweep' # attributes: # - attr: CollectionTime # field: gc.cms_collection_time # - attr: CollectionCount # field: gc.cms_collection_count #jmx.application: #jmx.instance: #-------------------------------- Kafka Module ------------------------------- #- module: kafka #metricsets: ["consumergroup", "partition"] #period: 10s #hosts: ["localhost:9092"] #enabled: true #client_id: metricbeat #retries: 3 #backoff: 250ms # List of Topics to query metadata for. If empty, all topics will be queried. #topics: [] # Optional SSL. By default is off. # List of root certificates for HTTPS server verifications #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # Certificate for SSL client authentication #ssl.certificate: "/etc/pki/client/cert.pem" # Client Certificate Key #ssl.key: "/etc/pki/client/cert.key" # SASL authentication #username: "" #password: "" #------------------------------- Kibana Module ------------------------------- {% if inventory_hostname in groups['kibana'] | default([]) %} - module: kibana metricsets: ["status"] period: 30s hosts: ["localhost:5601"] basepath: "" enabled: true # Set to true to send data collected by module to X-Pack # Monitoring instead of metricbeat-* indices. #xpack.enabled: false {% endif %} #----------------------------- Kubernetes Module ----------------------------- # Node metrics, from kubelet: #- module: kubernetes # metricsets: # - container # - node # - pod # - system # - volume # period: 10s # hosts: ["localhost:10255"] # enabled: true # #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token # #ssl.certificate_authorities: # # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt # #ssl.certificate: "/etc/pki/client/cert.pem" # #ssl.key: "/etc/pki/client/cert.key" # # # Enriching parameters: # add_metadata: true # in_cluster: true # # When used outside the cluster: # #host: node_name # #kube_config: ~/.kube/config # ## State metrics from kube-state-metrics service: #- module: kubernetes # enabled: true # metricsets: # - state_node # - state_deployment # - state_replicaset # - state_statefulset # - state_pod # - state_container # period: 10s # hosts: ["kube-state-metrics:8080"] # # # Enriching parameters: # add_metadata: true # in_cluster: true # # When used outside the cluster: # #host: node_name # #kube_config: ~/.kube/config # # Kubernetes events #- module: kubernetes # enabled: true # metricsets: # - event # # Kubernetes API server #- module: kubernetes # enabled: true # metricsets: # - apiserver # hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] #--------------------------------- kvm Module -------------------------------- {% if kvm_enabled | default(false) | bool %} - module: kvm metricsets: ["dommemstat"] enabled: true period: 10s hosts: ["unix:///var/run/libvirt/libvirt-sock"] # For remote hosts, setup network access in libvirtd.conf # and use the tcp scheme: # hosts: [ "tcp://:16509" ] # Timeout to connect to Libvirt server #timeout: 1s {% endif %} #------------------------------ Logstash Module ------------------------------ {% if inventory_hostname in groups['elastic-logstash'] | default([]) %} - module: logstash metricsets: ["node", "node_stats"] enabled: true period: 10s hosts: ["localhost:9600"] {% endif %} #------------------------------ Memcached Module ----------------------------- {% if memcached_enabled | default(false) | bool %} - module: memcached metricsets: ["stats"] enabled: true period: 30s hosts: ["{{ ansible_hostname }}:11211"] {% endif %} #------------------------------- MongoDB Module ------------------------------ #- module: mongodb # metricsets: ["dbstats", "status", "collstats", "metrics", "replstatus"] # period: 10s # enabled: true # # The hosts must be passed as MongoDB URLs in the format: # # [mongodb://][user:pass@]host[:port]. # # The username and password can also be set using the respective configuration # # options. The credentials in the URL take precedence over the username and # # password configuration options. # hosts: ["localhost:27017"] # # # Optional SSL. By default is off. # #ssl.enabled: true # # # Mode of verification of server certificate ('none' or 'full') # #ssl.verification_mode: 'full' # # # List of root certificates for TLS server verifications # #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # # # Certificate for SSL client authentication # #ssl.certificate: "/etc/pki/client/cert.pem" # # # Client Certificate Key # #ssl.key: "/etc/pki/client/cert.key" # # # Username to use when connecting to MongoDB. Empty by default. # #username: user # # # Password to use when connecting to MongoDB. Empty by default. # #password: pass # #-------------------------------- Munin Module ------------------------------- #- module: munin # metricsets: ["node"] # enabled: true # period: 10s # hosts: ["localhost:4949"] # List of plugins to collect metrics from, by default it collects from # all the available ones. #munin.plugins: [] # If set to true, it sanitizes fields names in concordance with munin # implementation (all characters that are not alphanumeric, or underscore # are replaced by underscores). #munin.sanitize: false #-------------------------------- MySQL Module ------------------------------- {% if (mysql_enabled | default(false) | bool) and galera_root_user is defined and galera_root_password is defined %} - module: mysql metricsets: - "status" # - "galera_status" enabled: true period: 30s # Host DSN should be defined as "user:pass@tcp(127.0.0.1:3306)/" # The username and password can either be set in the DSN or using the username # and password config options. Those specified in the DSN take precedence. hosts: ["{{ galera_root_user }}:{{ galera_root_password }}@tcp({{ ansible_hostname }}:3306)/"] # Username of hosts. Empty by default. username: {{ galera_root_user }} # Password of hosts. Empty by default. password: {{ galera_root_password }} # By setting raw to true, all raw fields from the status metricset will be added to the event. #raw: false {% endif %} #-------------------------------- Nats Module -------------------------------- {% if nats_enabled | bool %} - module: nats metricsets: ["connections", "routes", "stats", "subscriptions"] period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" #connections.metrics_path: "/connz" #routes.metrics_path: "/routez" #subscriptions.metrics_path: "/subsz" {% endif %} #-------------------------------- Nginx Module ------------------------------- {% if nginx_enabled | default(false) | bool %} - module: nginx metricsets: ["stubstatus"] enabled: true period: 30s # Nginx hosts hosts: ["http://127.0.1.1:18182"] # Path to server status. Default server-status server_status_path: "server-status" {% endif %} #------------------------------- PHP_FPM Module ------------------------------ #- module: php_fpm # metricsets: # - pool # #- process # enabled: true # period: 10s # status_path: "/status" # hosts: ["localhost:8080"] #----------------------------- PostgreSQL Module ----------------------------- #- module: postgresql # enabled: true # metricsets: # # Stats about every PostgreSQL database # - database # # # Stats about the background writer process's activity # - bgwriter # # # Stats about every PostgreSQL process # - activity # # period: 30s # # # The host must be passed as PostgreSQL URL. Example: # # postgres://localhost:5432?sslmode=disable # # The available parameters are documented here: # # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters # hosts: ["postgres://localhost:5432"] # # # Username to use when connecting to PostgreSQL. Empty by default. # #username: user # # # Password to use when connecting to PostgreSQL. Empty by default. # #password: pass #----------------------------- Prometheus Module ----------------------------- {% if (prometheus_enabled | default(false) | bool) and (prometheus_config is defined) %} {% for prometheus in prometheus_config %} - module: prometheus metricsets: [{% for mset in prometheus.metricsets|default(["collector"]) %}"{{ mset }}"{% if not loop.last %},{% endif %}{% endfor %}] enabled: {{ prometheus.enabled | default('true') }} period: {{ prometheus.period | default("10s") }} hosts: [{% for phost in prometheus.hosts %}"{{ phost }}"{% if not loop.last %},{% endif %}{% endfor %}] metrics_path: {{ prometheus.metrics_path | default("/metrics") }} namespace: {{ prometheus.namespace }} # {% endfor %} {% endif %} {% if (ceph_prometheus_enabled | default(false) | bool) %} - module: prometheus metricsets: ["collector"] enabled: 'true' period: "10s" hosts: [{% for phost in ceph_stats_hosts %}"{{ phost | regex_replace(':\\d+$', '') }}:9283"{% if not loop.last %},{% endif %}{% endfor %}] metrics_path: "/metrics" namespace: ceph {% endif %} # This can be used for service account based authorization: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt #------------------------------ RabbitMQ Module ------------------------------ {% if (rabbitmq_enabled | default(false) | bool) and (rabbitmq_monitoring_password is defined) %} - module: rabbitmq metricsets: ["node", "queue", "connection"] enabled: true period: 30s hosts: [ {{ elastic_metricbeat_rabbitmq_monitoring_hosts }} ] username: {{ rabbitmq_monitoring_userid | default('monitoring') }} password: {{ rabbitmq_monitoring_password }} {% endif %} # Management path prefix, if `management.path_prefix` is set in RabbitMQ # configuration, it has to be set to the same value. #management_path_prefix: "" #username: guest #password: guest #-------------------------------- Redis Module ------------------------------- #- module: redis # metricsets: ["info", "keyspace"] # enabled: true # period: 10s # # # Redis hosts # hosts: ["127.0.0.1:6379"] # # # Timeout after which time a metricset should return an error # # Timeout is by default defined as period, as a fetch of a metricset # # should never take longer then period, as otherwise calls can pile up. # #timeout: 1s # # # Optional fields to be added to each event # #fields: # # datacenter: west # # # Network type to be used for redis connection. Default: tcp # #network: tcp # # # Max number of concurrent connections. Default: 10 # #maxconn: 10 # # # Filters can be used to reduce the number of fields sent. #processors: # - include_fields: # fields: ["beat", "metricset", "redis.info.stats"] # Redis AUTH password. Empty by default. #password: foobared #------------------------------- traefik Module ------------------------------ {% if traefik_enabled | bool %} - module: traefik metricsets: ["health"] period: 10s hosts: ["localhost:8080"] {% endif %} #-------------------------------- uwsgi Module ------------------------------- {% if uwsgi_enabled | default(false) | bool %} - module: uwsgi metricsets: ["status"] enabled: true period: 30s {% if uwsgi_sockets.matched | int > 0 %} hosts: [{% for file in uwsgi_sockets.files %}"unix://{{ file.path }}"{% if not loop.last %},{% endif %}{% endfor %}] {% else %} hosts: ["tcp://127.0.0.1:9191"] {% endif %} {% endif %} #------------------------------- vSphere Module ------------------------------ #- module: vsphere # enabled: true # metricsets: ["datastore", "host", "virtualmachine"] # period: 10s # hosts: ["https://localhost/sdk"] # # username: "user" # password: "password" # # If insecure is true, don't verify the server's certificate chain # insecure: false # # Get custom fields when using virtualmachine metric set. Default false. # # get_custom_fields: false #------------------------------- Windows Module ------------------------------ #- module: windows # metricsets: ["perfmon"] # enabled: true # period: 10s # perfmon.ignore_non_existent_counters: true # perfmon.counters: # # - instance_label: processor.name # # instance_name: total # # measurement_label: processor.time.total.pct # # query: '\Processor Information(_Total)\% Processor Time' # #- module: windows # metricsets: ["service"] # enabled: true # period: 60s # #------------------------------ ZooKeeper Module ----------------------------- #- module: zookeeper # enabled: true # metricsets: ["mntr", "server"] # period: 10s # hosts: ["localhost:2181"] #================================ General ====================================== # The name of the shipper that publishes the network data. It can be used to group # all the transactions sent by a single shipper in the web interface. # If this options is not defined, the hostname is used. #name: # The tags of the shipper are included in their own field with each # transaction published. Tags make it easy to group servers by different # logical properties. #tags: ["service-X", "web-tier"] # Optional fields that you can specify to add additional information to the # output. Fields can be scalar values, arrays, dictionaries, or any nested # combination of these. #fields: # env: staging # If this option is set to true, the custom fields are stored as top-level # fields in the output document instead of being grouped under a fields # sub-dictionary. Default is false. #fields_under_root: false # Internal queue configuration for buffering events to be published. #queue: # Queue type by name (default 'mem') # The memory queue will present all available events (up to the outputs # bulk_max_size) to the output, the moment the output is ready to server # another batch of events. #mem: # Max number of events the queue can buffer. #events: 4096 # Hints the minimum number of events stored in the queue, # before providing a batch of events to the outputs. # The default value is set to 2048. # A value of 0 ensures events are immediately available # to be sent to the outputs. #flush.min_events: 2048 # Maximum duration after which events are available to the outputs, # if the number of events stored in the queue is < min_flush_events. #flush.timeout: 1s # The spool queue will store events in a local spool file, before # forwarding the events to the outputs. # # Beta: spooling to disk is currently a beta feature. Use with care. # # The spool file is a circular buffer, which blocks once the file/buffer is full. # Events are put into a write buffer and flushed once the write buffer # is full or the flush_timeout is triggered. # Once ACKed by the output, events are removed immediately from the queue, # making space for new events to be persisted. #spool: # The file namespace configures the file path and the file creation settings. # Once the file exists, the `size`, `page_size` and `prealloc` settings # will have no more effect. #file: # Location of spool file. The default value is ${path.data}/spool.dat. #path: "${path.data}/spool.dat" # Configure file permissions if file is created. The default value is 0600. #permissions: 0600 # File size hint. The spool blocks, once this limit is reached. The default value is 100 MiB. #size: 100MiB # The files page size. A file is split into multiple pages of the same size. The default value is 4KiB. #page_size: 4KiB # If prealloc is set, the required space for the file is reserved using # truncate. The default value is true. #prealloc: true # Spool writer settings # Events are serialized into a write buffer. The write buffer is flushed if: # - The buffer limit has been reached. # - The configured limit of buffered events is reached. # - The flush timeout is triggered. #write: # Sets the write buffer size. #buffer_size: 1MiB # Maximum duration after which events are flushed if the write buffer # is not full yet. The default value is 1s. #flush.timeout: 1s # Number of maximum buffered events. The write buffer is flushed once the # limit is reached. #flush.events: 16384 # Configure the on-disk event encoding. The encoding can be changed # between restarts. # Valid encodings are: json, ubjson, and cbor. #codec: cbor #read: # Reader flush timeout, waiting for more events to become available, so # to fill a complete batch as required by the outputs. # If flush_timeout is 0, all available events are forwarded to the # outputs immediately. # The default value is 0s. #flush.timeout: 0s # Sets the maximum number of CPUs that can be executing simultaneously. The # default is the number of logical CPUs available in the system. #max_procs: #================================ Processors =================================== # Processors are used to reduce the number of fields in the exported event or to # enhance the event with external metadata. This section defines a list of # processors that are applied one by one and the first one receives the initial # event: # # event -> filter1 -> event1 -> filter2 ->event2 ... # # The supported processors are drop_fields, drop_event, include_fields, # decode_json_fields, and add_cloud_metadata. # # For example, you can use the following processors to keep the fields that # contain CPU load percentages, but remove the fields that contain CPU ticks # values: # #processors: #- include_fields: # fields: ["cpu"] #- drop_fields: # fields: ["cpu.user", "cpu.system"] # # The following example drops the events that have the HTTP response code 200: # #processors: #- drop_event: # when: # equals: # http.code: 200 # # The following example renames the field a to b: # #processors: #- rename: # fields: # - from: "a" # to: "b" # # The following example tokenizes the string into fields: # #processors: #- dissect: # tokenizer: "%{key1} - %{key2}" # field: "message" # target_prefix: "dissect" # # The following example enriches each event with metadata from the cloud # provider about the host machine. It works on EC2, GCE, DigitalOcean, # Tencent Cloud, and Alibaba Cloud. # #processors: #- add_cloud_metadata: ~ # # The following example enriches each event with the machine's local time zone # offset from UTC. # #processors: #- add_locale: # format: offset # # The following example enriches each event with docker metadata, it matches # given fields to an existing container id and adds info from that container: # #processors: #- add_docker_metadata: # host: "unix:///var/run/docker.sock" # match_fields: ["system.process.cgroup.id"] # match_pids: ["process.pid", "process.ppid"] # match_source: true # match_source_index: 4 # match_short_id: false # cleanup_timeout: 60 # labels.dedot: false # # To connect to Docker over TLS you must specify a client and CA certificate. # #ssl: # # certificate_authority: "/etc/pki/root/ca.pem" # # certificate: "/etc/pki/client/cert.pem" # # key: "/etc/pki/client/cert.key" # # The following example enriches each event with docker metadata, it matches # container id from log path available in `source` field (by default it expects # it to be /var/lib/docker/containers/*/*.log). # #processors: #- add_docker_metadata: ~ # # The following example enriches each event with host metadata. # #processors: #- add_host_metadata: # netinfo.enabled: false # # The following example enriches each event with process metadata using # process IDs included in the event. # #processors: #- add_process_metadata: # match_pids: ["system.process.ppid"] # target: system.process.parent # # The following example decodes fields containing JSON strings # and replaces the strings with valid JSON objects. # #processors: #- decode_json_fields: # fields: ["field1", "field2", ...] # process_array: false # max_depth: 1 # target: "" # overwrite_keys: false processors: - add_host_metadata: ~ #============================= Elastic Cloud ================================== # These settings simplify using metricbeat with the Elastic Cloud (https://cloud.elastic.co/). # The cloud.id setting overwrites the `output.elasticsearch.hosts` and # `setup.kibana.host` options. # You can find the `cloud.id` in the Elastic Cloud web UI. #cloud.id: # The cloud.auth setting overwrites the `output.elasticsearch.username` and # `output.elasticsearch.password` settings. The format is `:`. #cloud.auth: #================================ Outputs ====================================== # Configure what output to use when sending the data collected by the beat. #-------------------------- Elasticsearch output ------------------------------- #output.elasticsearch: # # Boolean flag to enable or disable the output module. # #enabled: true # # # Array of hosts to connect to. # # Scheme and port can be left out and will be set to the default (http and 9200) # # In case you specify and additional path, the scheme is required: http://localhost:9200/path # # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 # hosts: ["localhost:9200"] # # # Set gzip compression level. # #compression_level: 0 # # # Configure escaping HTML symbols in strings. # #escape_html: false # # # Optional protocol and basic auth credentials. # #protocol: "https" # #username: "elastic" # #password: "changeme" # # # Dictionary of HTTP parameters to pass within the URL with index operations. # #parameters: # #param1: value1 # #param2: value2 # # # Number of workers per Elasticsearch host. # #worker: 1 # # # Optional index name. The default is "metricbeat" plus date # # and generates [metricbeat-]YYYY.MM.DD keys. # # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. # #index: "metricbeat-%{[agent.version]}-%{+yyyy.MM.dd}" # # # Optional ingest node pipeline. By default no pipeline will be used. # #pipeline: "" # # # Optional HTTP path # #path: "/elasticsearch" # # # Custom HTTP headers to add to each request # #headers: # # X-My-Header: Contents of the header # # # Proxy server URL # #proxy_url: http://proxy:3128 # # # The number of times a particular Elasticsearch index operation is attempted. If # # the indexing operation doesn't succeed after this many retries, the events are # # dropped. The default is 3. # #max_retries: 3 # # # The maximum number of events to bulk in a single Elasticsearch bulk API index request. # # The default is 50. # #bulk_max_size: 50 # # # The number of seconds to wait before trying to reconnect to Elasticsearch # # after a network error. After waiting backoff.init seconds, the Beat # # tries to reconnect. If the attempt fails, the backoff timer is increased # # exponentially up to backoff.max. After a successful connection, the backoff # # timer is reset. The default is 1s. # #backoff.init: 1s # # # The maximum number of seconds to wait before attempting to connect to # # Elasticsearch after a network error. The default is 60s. # #backoff.max: 60s # # # Configure HTTP request timeout before failing a request to Elasticsearch. # #timeout: 90 # # # Use SSL settings for HTTPS. # #ssl.enabled: true # # # Configure SSL verification mode. If `none` is configured, all server hosts # # and certificates will be accepted. In this mode, SSL-based connections are # # susceptible to man-in-the-middle attacks. Use only for testing. Default is # # `full`. # #ssl.verification_mode: full # # # List of supported/valid TLS versions. By default all TLS versions from 1.0 up to # # 1.2 are enabled. # #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] # # # List of root certificates for HTTPS server verifications # #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # # # Certificate for SSL client authentication # #ssl.certificate: "/etc/pki/client/cert.pem" # # # Client certificate key # #ssl.key: "/etc/pki/client/cert.key" # # # Optional passphrase for decrypting the certificate key. # #ssl.key_passphrase: '' # # # Configure cipher suites to be used for SSL connections # #ssl.cipher_suites: [] # # # Configure curve types for ECDHE-based cipher suites # #ssl.curve_types: [] # # # Configure what types of renegotiation are supported. Valid options are # # never, once, and freely. Default is never. # #ssl.renegotiation: never #----------------------------- Logstash output --------------------------------- {{ elk_macros.output_logstash(inventory_hostname, logstash_data_hosts, ansible_processor_count) }} #------------------------------- Kafka output ---------------------------------- #output.kafka: # Boolean flag to enable or disable the output module. #enabled: true # The list of Kafka broker addresses from which to fetch the cluster metadata. # The cluster metadata contain the actual Kafka brokers events are published # to. #hosts: ["localhost:9092"] # The Kafka topic used for produced events. The setting can be a format string # using any event field. To set the topic from document type use `%{[type]}`. #topic: beats # The Kafka event key setting. Use format string to create a unique event key. # By default no event key will be generated. #key: '' # The Kafka event partitioning strategy. Default hashing strategy is `hash` # using the `output.kafka.key` setting or randomly distributes events if # `output.kafka.key` is not configured. #partition.hash: # If enabled, events will only be published to partitions with reachable # leaders. Default is false. #reachable_only: false # Configure alternative event field names used to compute the hash value. # If empty `output.kafka.key` setting will be used. # Default value is empty list. #hash: [] # Authentication details. Password is required if username is set. #username: '' #password: '' # Kafka version metricbeat is assumed to run against. Defaults to the "1.0.0". #version: '1.0.0' # Configure JSON encoding #codec.json: # Pretty-print JSON event #pretty: false # Configure escaping HTML symbols in strings. #escape_html: false # Metadata update configuration. Metadata contains leader information # used to decide which broker to use when publishing. #metadata: # Max metadata request retry attempts when cluster is in middle of leader # election. Defaults to 3 retries. #retry.max: 3 # Wait time between retries during leader elections. Default is 250ms. #retry.backoff: 250ms # Refresh metadata interval. Defaults to every 10 minutes. #refresh_frequency: 10m # Strategy for fetching the topics metadata from the broker. Default is true. #full: true # The number of concurrent load-balanced Kafka output workers. #worker: 1 # The number of times to retry publishing an event after a publishing failure. # After the specified number of retries, events are typically dropped. # Some Beats, such as Filebeat, ignore the max_retries setting and retry until # all events are published. Set max_retries to a value less than 0 to retry # until all events are published. The default is 3. #max_retries: 3 # The maximum number of events to bulk in a single Kafka request. The default # is 2048. #bulk_max_size: 2048 # The number of seconds to wait for responses from the Kafka brokers before # timing out. The default is 30s. #timeout: 30s # The maximum duration a broker will wait for number of required ACKs. The # default is 10s. #broker_timeout: 10s # The number of messages buffered for each Kafka broker. The default is 256. #channel_buffer_size: 256 # The keep-alive period for an active network connection. If 0s, keep-alives # are disabled. The default is 0 seconds. #keep_alive: 0 # Sets the output compression codec. Must be one of none, snappy and gzip. The # default is gzip. #compression: gzip # Set the compression level. Currently only gzip provides a compression level # between 0 and 9. The default value is chosen by the compression algorithm. #compression_level: 4 # The maximum permitted size of JSON-encoded messages. Bigger messages will be # dropped. The default value is 1000000 (bytes). This value should be equal to # or less than the broker's message.max.bytes. #max_message_bytes: 1000000 # The ACK reliability level required from broker. 0=no response, 1=wait for # local commit, -1=wait for all replicas to commit. The default is 1. Note: # If set to 0, no ACKs are returned by Kafka. Messages might be lost silently # on error. #required_acks: 1 # The configurable ClientID used for logging, debugging, and auditing # purposes. The default is "beats". #client_id: beats # Enable SSL support. SSL is automatically enabled if any SSL setting is set. #ssl.enabled: true # Optional SSL configuration options. SSL is off by default. # List of root certificates for HTTPS server verifications #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # Configure SSL verification mode. If `none` is configured, all server hosts # and certificates will be accepted. In this mode, SSL based connections are # susceptible to man-in-the-middle attacks. Use only for testing. Default is # `full`. #ssl.verification_mode: full # List of supported/valid TLS versions. By default all TLS versions from 1.0 up to # 1.2 are enabled. #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] # Certificate for SSL client authentication #ssl.certificate: "/etc/pki/client/cert.pem" # Client Certificate Key #ssl.key: "/etc/pki/client/cert.key" # Optional passphrase for decrypting the Certificate Key. #ssl.key_passphrase: '' # Configure cipher suites to be used for SSL connections #ssl.cipher_suites: [] # Configure curve types for ECDHE-based cipher suites #ssl.curve_types: [] # Configure what types of renegotiation are supported. Valid options are # never, once, and freely. Default is never. #ssl.renegotiation: never #------------------------------- Redis output ---------------------------------- #output.redis: # Boolean flag to enable or disable the output module. #enabled: true # Configure JSON encoding #codec.json: # Pretty print json event #pretty: false # Configure escaping HTML symbols in strings. #escape_html: false # The list of Redis servers to connect to. If load-balancing is enabled, the # events are distributed to the servers in the list. If one server becomes # unreachable, the events are distributed to the reachable servers only. #hosts: ["localhost:6379"] # The name of the Redis list or channel the events are published to. The # default is metricbeat. #key: metricbeat # The password to authenticate to Redis with. The default is no authentication. #password: # The Redis database number where the events are published. The default is 0. #db: 0 # The Redis data type to use for publishing events. If the data type is list, # the Redis RPUSH command is used. If the data type is channel, the Redis # PUBLISH command is used. The default value is list. #datatype: list # The number of workers to use for each host configured to publish events to # Redis. Use this setting along with the loadbalance option. For example, if # you have 2 hosts and 3 workers, in total 6 workers are started (3 for each # host). #worker: 1 # If set to true and multiple hosts or workers are configured, the output # plugin load balances published events onto all Redis hosts. If set to false, # the output plugin sends all events to only one host (determined at random) # and will switch to another host if the currently selected one becomes # unreachable. The default value is true. #loadbalance: true # The Redis connection timeout in seconds. The default is 5 seconds. #timeout: 5s # The number of times to retry publishing an event after a publishing failure. # After the specified number of retries, the events are typically dropped. # Some Beats, such as Filebeat, ignore the max_retries setting and retry until # all events are published. Set max_retries to a value less than 0 to retry # until all events are published. The default is 3. #max_retries: 3 # The number of seconds to wait before trying to reconnect to Redis # after a network error. After waiting backoff.init seconds, the Beat # tries to reconnect. If the attempt fails, the backoff timer is increased # exponentially up to backoff.max. After a successful connection, the backoff # timer is reset. The default is 1s. #backoff.init: 1s # The maximum number of seconds to wait before attempting to connect to # Redis after a network error. The default is 60s. #backoff.max: 60s # The maximum number of events to bulk in a single Redis request or pipeline. # The default is 2048. #bulk_max_size: 2048 # The URL of the SOCKS5 proxy to use when connecting to the Redis servers. The # value must be a URL with a scheme of socks5://. #proxy_url: # This option determines whether Redis hostnames are resolved locally when # using a proxy. The default value is false, which means that name resolution # occurs on the proxy server. #proxy_use_local_resolver: false # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. #ssl.enabled: true # Configure SSL verification mode. If `none` is configured, all server hosts # and certificates will be accepted. In this mode, SSL based connections are # susceptible to man-in-the-middle attacks. Use only for testing. Default is # `full`. #ssl.verification_mode: full # List of supported/valid TLS versions. By default all TLS versions 1.0 up to # 1.2 are enabled. #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] # Optional SSL configuration options. SSL is off by default. # List of root certificates for HTTPS server verifications #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] # Certificate for SSL client authentication #ssl.certificate: "/etc/pki/client/cert.pem" # Client Certificate Key #ssl.key: "/etc/pki/client/cert.key" # Optional passphrase for decrypting the Certificate Key. #ssl.key_passphrase: '' # Configure cipher suites to be used for SSL connections #ssl.cipher_suites: [] # Configure curve types for ECDHE based cipher suites #ssl.curve_types: [] # Configure what types of renegotiation are supported. Valid options are # never, once, and freely. Default is never. #ssl.renegotiation: never #------------------------------- File output ----------------------------------- #output.file: # Boolean flag to enable or disable the output module. #enabled: true # Configure JSON encoding #codec.json: # Pretty-print JSON event #pretty: false # Configure escaping HTML symbols in strings. #escape_html: false # Path to the directory where to save the generated files. The option is # mandatory. #path: "/tmp/metricbeat" # Name of the generated files. The default is `metricbeat` and it generates # files: `metricbeat`, `metricbeat.1`, `metricbeat.2`, etc. #filename: metricbeat # Maximum size in kilobytes of each file. When this size is reached, and on # every metricbeat restart, the files are rotated. The default value is 10240 # kB. #rotate_every_kb: 10000 # Maximum number of files under path. When this number of files is reached, # the oldest file is deleted and the rest are shifted from last to first. The # default is 7 files. #number_of_files: 7 # Permissions to use for file creation. The default is 0600. #permissions: 0600 #----------------------------- Console output --------------------------------- #output.console: # Boolean flag to enable or disable the output module. #enabled: true # Configure JSON encoding #codec.json: # Pretty-print JSON event #pretty: false # Configure escaping HTML symbols in strings. #escape_html: false #================================= Paths ====================================== # The home path for the metricbeat installation. This is the default base path # for all other path settings and for miscellaneous files that come with the # distribution (for example, the sample dashboards). # If not set by a CLI flag or in the configuration file, the default for the # home path is the location of the binary. #path.home: # The configuration path for the metricbeat installation. This is the default # base path for configuration files, including the main YAML configuration file # and the Elasticsearch template file. If not set by a CLI flag or in the # configuration file, the default for the configuration path is the home path. #path.config: ${path.home} # The data path for the metricbeat installation. This is the default base path # for all the files in which metricbeat needs to store its data. If not set by a # CLI flag or in the configuration file, the default for the data path is a data # subdirectory inside the home path. #path.data: ${path.home}/data # The logs path for a metricbeat installation. This is the default location for # the Beat's log files. If not set by a CLI flag or in the configuration file, # the default for the logs path is a logs subdirectory inside the home path. #path.logs: ${path.home}/logs #================================ Keystore ========================================== # Location of the Keystore containing the keys and their sensitive values. #keystore.path: "${path.config}/beats.keystore" #============================== Dashboards ===================================== {% if (groups['kibana'] | length) > 0 and (setup_kibana_dashboards | default(true)) %} {{ elk_macros.setup_dashboards('metricbeat') }} {% endif %} #============================== Template ===================================== {{ elk_macros.setup_template('metricbeat', inventory_hostname, data_nodes, elasticsearch_beat_settings) }} #============================== Setup ILM ===================================== # Configure Index Lifecycle Management Index Lifecycle Management creates a # write alias and adds additional settings to the template. # The elasticsearch.output.index setting will be replaced with the write alias # if ILM is enabled. # Enabled ILM support. Valid values are true, false, and auto. The beat will # detect availabilty of Index Lifecycle Management in Elasticsearch and enable # or disable ILM support. #setup.ilm.enabled: auto # Configure the ILM write alias name. #setup.ilm.rollover_alias: "metricbeat" # Configure rollover index pattern. #setup.ilm.pattern: "{now/d}-000001" {% if ilm_policy_name is defined %} setup.ilm.policy_name: "{{ ilm_policy_name }}" {% endif %} {% if ilm_policy_file_location is defined %} setup.ilm.policy_file: "{{ ilm_policy_file_location }}/{{ ilm_policy_filename }}" {% endif %} #============================== Kibana ===================================== {% if (groups['kibana'] | length) > 0 and (setup_kibana_dashboards | default(true)) %} {{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }} {% endif %} #================================ Logging ====================================== {{ elk_macros.beat_logging('metricbeat') }} #============================== Xpack Monitoring ===================================== {{ elk_macros.xpack_monitoring_elasticsearch(inventory_hostname, elasticsearch_data_hosts, ansible_processor_count) }} #================================ HTTP Endpoint ====================================== # Each beat can expose internal metrics through a HTTP endpoint. For security # reasons the endpoint is disabled by default. This feature is currently experimental. # Stats can be access through http://localhost:5066/stats . For pretty JSON output # append ?pretty to the URL. # Defines if the HTTP endpoint is enabled. #http.enabled: false # The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost. #http.host: localhost # Port on which the HTTP endpoint will bind. Default is 5066. #http.port: 5066 #============================= Process Security ================================ # Enable or disable seccomp system call filtering on Linux. Default is enabled. #seccomp.enabled: true #================================= Migration ================================== # This allows to enable 6.7 migration aliases #migration.6_to_7.enabled: false