Skip to content

Collector Config

Collector is configured with a TOML-formatted file. In Kubernetes deployments, this is typically within a ConfigMap mounted into the collector pod. A default config can be generated by running ./collector config.

Global Config

This is the top level configuration for the collector. The only required fields are Endpoint and StorageDir.

# Ingestor URL to send collected telemetry.
endpoint = 'https://ingestor.adx-mon.svc.cluster.local'
# Path to kubernetes client config
kube-config = '.kube/config'
# Skip TLS verification.
insecure-skip-verify = true
# Address to listen on for endpoints.
listen-addr = ':8080'
# Region is a location identifier.
region = 'eastus'
# Optional path to the TLS key file.
tls-key-file = '/etc/certs/collector.key'
# Optional path to the TLS cert bundle file.
tls-cert-file = '/etc/certs/collector.pem'
# Maximum number of connections to accept.
max-connections = 100
# Maximum number of samples to send in a single batch.
max-batch-size = 1000
# Max segment agent in seconds.
max-segment-age-seconds = 30
# Maximum segment size in bytes.
max-segment-size = 52428800
# Maximum allowed size in bytes of all segments on disk.
max-disk-usage = 53687091200
# Interval to flush the WAL. (default 100)
wal-flush-interval-ms = 100
# Maximum number of concurrent transfers.
max-transfer-concurrency = 100
# Storage directory for the WAL and log cursors.
storage-dir = '/var/lib/adx-mon'
# Enable pprof endpoints.
enable-pprof = true
# Default to dropping all metrics.  Only metrics matching a keep rule will be kept.
default-drop-metrics = false
# Global Regexes of metrics to drop.
drop-metrics = [
  '^kube_pod_ips$',
  'etcd_grpc.*'
]
# Global Regexes of metrics to keep.
keep-metrics = [
  'nginx.*'
]
# Attributes lifted from the Body field and added to Attributes.
lift-attributes = [
  'host'
]

# Global Key/value pairs of labels to add to all metrics.
[add-labels]
  collectedBy = 'collector'

# Global labels to drop if they match a metrics regex in the format <metrics regex>=<label name>. These are dropped from all metrics collected by this agent
[drop-labels]
  '^nginx_connections_accepted' = '^pid$'

# Global Regexes of metrics to keep if they have the given label and value. These are kept from all metrics collected by this agent
[[keep-metrics-with-label-value]]
  # The regex to match the label value against.  If the label value matches, the metric will be kept.
  label-regex = 'owner'
  # The regex to match the label value against.  If the label value matches, the metric will be kept.
  value-regex = 'platform'

[[keep-metrics-with-label-value]]
  # The regex to match the label value against.  If the label value matches, the metric will be kept.
  label-regex = 'type'
  # The regex to match the label value against.  If the label value matches, the metric will be kept.
  value-regex = 'frontend|backend'

# Global labels to lift from the metric to top level columns
[[lift-labels]]
  # The name of the label to lift.
  name = 'Host'
  # The name of the column to lift the label to.
  column = ''

[[lift-labels]]
  # The name of the label to lift.
  name = 'cluster_name'
  # The name of the column to lift the label to.
  column = 'Cluster'

# Key/value pairs of attributes to add to all logs.
[add-attributes]
  cluster = 'cluster1'
  geo = 'eu'

# Optional configuration for exporting telemetry outside of adx-mon in parallel with sending to ADX.
# Exporters are declared here and referenced by name in each collection source.
[exporters]
  # Configuration for exporting metrics to an OTLP/HTTP endpoint.
  [[exporters.otlp-metric-export]]
    # Name of the exporter.
    name = 'to-otlp'
    # OTLP/HTTP endpoint to send metrics to.
    destination = 'http://localhost:4318/v1/metrics'
    # Default to dropping all metrics.  Only metrics matching a keep rule will be kept.
    default-drop-metrics = true
    # Regexes of metrics to drop.
    drop-metrics = []
    # Regexes of metrics to keep.
    keep-metrics = [
      '^kube_pod_ips$'
    ]
    # Regexes of metrics to keep if they have the given label and value.
    keep-metrics-with-label-value = []

    # Key/value pairs of labels to add to all metrics.
    [exporters.otlp-metric-export.add-labels]
      forwarded_to = 'otlp'

    # Labels to drop if they match a metrics regex in the format <metrics regex>=<label name>.
    [exporters.otlp-metric-export.drop-labels]
      '^kube_pod_ips$' = '^ip_family'

    # Key/value pairs of resource attributes to add to all metrics.
    [exporters.otlp-metric-export.add-resource-attributes]
      destination_namespace = 'prod-metrics'

Prometheus Scrape

Prometheus scrape discovers pods with the adx-mon/scrape annotation as well as any defined static scrape targets. It ships any metrics to the defined ADX database.

# Defines a prometheus format endpoint scraper.
[prometheus-scrape]
  # Database to store metrics in.
  database = 'Metrics'
  # Scrape interval in seconds.
  scrape-interval = 10
  # Scrape timeout in seconds.
  scrape-timeout = 5
  # Disable metrics forwarding to endpoints.
  disable-metrics-forwarding = false
  # Disable discovery of kubernetes pod targets.
  disable-discovery = false
  # Regexes of metrics to drop.
  drop-metrics = [
    '^kube_pod_ips$',
    'etcd_grpc.*'
  ]
  # Regexes of metrics to keep.
  keep-metrics = [
    'nginx.*'
  ]
  # List of exporter names to forward metrics to.
  exporters = []

  # Defines a static scrape target.
  [[prometheus-scrape.static-scrape-target]]
    # The regex to match the host name against.  If the hostname matches, the URL will be scraped.
    host-regex = '.*'
    # The URL to scrape.
    url = 'http://localhost:9090/metrics'
    # The namespace label to add for metrics scraped at this URL.
    namespace = 'monitoring'
    # The pod label to add for metrics scraped at this URL.
    pod = 'host-monitor'
    # The container label to add for metrics scraped at this URL.
    container = 'host-monitor'

  # Regexes of metrics to keep if they have the given label and value.
  [[prometheus-scrape.keep-metrics-with-label-value]]
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    label-regex = 'owner'
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    value-regex = 'platform'

  [[prometheus-scrape.keep-metrics-with-label-value]]
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    label-regex = 'type'
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    value-regex = 'frontend|backend'

Prometheus Remote Write

Prometheus remote write accepts metrics from Prometheus remote write protocol. It ships metrics to the defined ADX database.

# Defines a prometheus remote write endpoint.
[[prometheus-remote-write]]
  # Database to store metrics in.
  database = 'Metrics'
  # The path to listen on for prometheus remote write requests.  Defaults to /receive.
  path = '/receive'
  # Regexes of metrics to drop.
  drop-metrics = [
    '^kube_pod_ips$',
    'etcd_grpc.*'
  ]
  # Regexes of metrics to keep.
  keep-metrics = [
    'nginx.*'
  ]
  # List of exporter names to forward metrics to.
  exporters = []

  # Key/value pairs of labels to add to all metrics.
  [prometheus-remote-write.add-labels]
    cluster = 'cluster1'

  # Labels to drop if they match a metrics regex in the format <metrics regex>=<label name>.
  [prometheus-remote-write.drop-labels]
    '^nginx_connections_accepted' = '^pid$'

  # Regexes of metrics to keep if they have the given label and value.
  [[prometheus-remote-write.keep-metrics-with-label-value]]
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    label-regex = 'owner'
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    value-regex = 'platform'

  [[prometheus-remote-write.keep-metrics-with-label-value]]
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    label-regex = 'type'
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    value-regex = 'frontend|backend'

Otel Log

The Otel log endpoint accepts OTLP/HTTP logs from an OpenTelemetry sender. By default, this listens under the path /v1/logs.

# Defines an OpenTelemetry log endpoint. Accepts OTLP/HTTP.
[otel-log]
  # Attributes lifted from the Body and added to Attributes.
  lift-attributes = [
    'host'
  ]
  # List of exporter names to forward logs to.
  exporters = []

  # Key/value pairs of attributes to add to all logs.
  [otel-log.add-attributes]
    cluster = 'cluster1'
    geo = 'eu'

  # Defines a list of transforms to apply to log lines.
  [[otel-log.transforms]]
    # The name of the transform to apply to the log line.
    name = 'addattributes'

    # The configuration for the transform.
    [otel-log.transforms.config]
      environment = 'production'

Otel Metrics

The Otel metrics endpoint accepts OTLP/HTTP and/or OTLP/gRPC metrics from an OpenTelemetry sender.

# Defines an OpenTelemetry metric endpoint. Accepts OTLP/HTTP and/or OTLP/gRPC.
[[otel-metric]]
  # Database to store metrics in.
  database = 'Metrics'
  # The path to listen on for OTLP/HTTP requests.
  path = '/v1/otlpmetrics'
  # The port to listen on for OTLP/gRPC requests.
  grpc-port = 4317
  # Regexes of metrics to drop.
  drop-metrics = [
    '^kube_pod_ips$',
    'etcd_grpc.*'
  ]
  # Regexes of metrics to keep.
  keep-metrics = [
    'nginx.*'
  ]
  # List of exporter names to forward metrics to.
  exporters = []

  # Key/value pairs of labels to add to all metrics.
  [otel-metric.add-labels]
    cluster = 'cluster1'

  # Labels to drop if they match a metrics regex in the format <metrics regex>=<label name>.  These are dropped from all metrics collected by this agent
  [otel-metric.drop-labels]
    '^nginx_connections_accepted' = '^pid$'

  # Regexes of metrics to keep if they have the given label and value.
  [[otel-metric.keep-metrics-with-label-value]]
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    label-regex = 'owner'
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    value-regex = 'platform'

  [[otel-metric.keep-metrics-with-label-value]]
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    label-regex = 'type'
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    value-regex = 'frontend|backend'

Host Log

The host log config configures file and journald log collection. By default, Kubernetes pods with adx-mon/log-destination annotation will have their logs scraped and sent to the appropriate destinations.

Log Parsers

Parsers are used within file-target and journal-target configurations to process the raw log message extracted from the source (e.g., a file line or a journald entry). They are defined in the parsers array and are applied sequentially.

The parsers array accepts a list of strings, each specifying a parser type. The collector attempts to apply each parser in the order they are listed. The first parser that successfully processes the log message stops the parsing process for that message. If a parser succeeds, the resulting fields are added to the log's body.

If no parser in the list succeeds, the original raw log message is kept in the message field of the log body.

Available parser types:

  • json: Attempts to parse the entire log message string as a JSON object. If successful, the key-value pairs from the JSON object are merged into the log body. The original message field is typically removed or overwritten by a field from the JSON payload if one exists with the key "message".
  • keyvalue: Parses log messages formatted as key1=value1 key2="quoted value" key3=value3 .... It extracts these key-value pairs and adds them to the log body. Keys and values are strings. Values containing spaces should be quoted.
  • space: Splits the log message string by whitespace (using strings.Fields, which handles multiple spaces, tabs, etc.). Each resulting part is added to the log body with keys named sequentially: field0, field1, field2, and so on. All resulting fields are strings.
# Defines a host log scraper.
[[host-log]]
  # Disable discovery of Kubernetes pod targets. Only one HostLog configuration can use Kubernetes discovery.
  disable-kube-discovery = true
  # List of exporter names to forward logs to.
  exporters = []

  # Defines a tail file target.
  [[host-log.file-target]]
    # The path to the file to tail.
    file-path = '/var/log/nginx/access.log'
    # The type of log being output. This defines how timestamps and log messages are extracted from structured log types like docker json files. Options are: docker, plain.
    log-type = 'plain'
    # Database to store logs in.
    database = 'Logs'
    # Table to store logs in.
    table = 'NginxAccess'
    # Parsers to apply sequentially to the log line.
    parsers = []

  [[host-log.file-target]]
    # The path to the file to tail.
    file-path = '/var/log/myservice/service.log'
    # The type of log being output. This defines how timestamps and log messages are extracted from structured log types like docker json files. Options are: docker, plain.
    log-type = 'plain'
    # Database to store logs in.
    database = 'Logs'
    # Table to store logs in.
    table = 'NginxAccess'
    # Parsers to apply sequentially to the log line.
    parsers = [
      'json'
    ]

  # Defines a static Kubernetes pod target to scrape. These are pods managed by the Kubelet and not discoverable via the apiserver.
  [[host-log.static-pod-target]]
    # The namespace of the pod to scrape.
    namespace = 'default'
    # The name of the pod to scrape.
    name = 'myapp'
    # The destination to send the logs to.  Syntax matches that of adx-mon/log-destination annotations.
    destination = 'Logs:MyApp'
    # Parsers to apply sequentially to the log line.
    parsers = [
      'json'
    ]

    # The labels to match on the pod.  If the pod has all of these labels, it will be scraped.
    [host-log.static-pod-target.label-targets]
      app = 'myapp'

  # Defines a journal target to scrape.
  [[host-log.journal-target]]
    # Matches for the journal reader based on journalctl MATCHES. To select a systemd unit, use the field _SYSTEMD_UNIT. (e.g. '_SYSTEMD_UNIT=avahi-daemon.service' for selecting logs from the avahi-daemon service.)
    matches = [
      '_SYSTEMD_UNIT=docker.service',
      '_TRANSPORT=journal'
    ]
    # Database to store logs in.
    database = 'Logs'
    # Table to store logs in.
    table = 'Docker'
    # Parsers to apply sequentially to the log line.
    parsers = []
    # Optional journal metadata fields http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
    journal-fields = []

  # Defines a kernel target to scrape.
  [[host-log.kernel-target]]
    # Database to store logs in.
    database = 'Logs'
    # Table to store logs in.
    table = 'Kernel'
    # One of emerg, alert, crit, err, warning, notice, info, debug, default is info.
    priority = 'warning'

  # Defines a list of transforms to apply to log lines.
  [[host-log.transforms]]
    # The name of the transform to apply to the log line.
    name = 'addattributes'

    # The configuration for the transform.
    [host-log.transforms.config]
      environment = 'production'

Exporters

Exporters are used to send telemetry to external systems in parallel with data sent to Azure Data Explorer. Exporters are per-source type (e.g. Metrics, Logs). Exporters are defined under the top level configuration key [exporters] within a key representing the exporter type (e.g. [exporters.otlp-metric-export]). They are referenced by their configured name in the relevant telemetry collection section.

Metric Exporters

Metrics currently support exporting to OpenTelemetry OTLP/HTTP endpoints with otlp-metric-exporter. The exporter can be configured to drop metrics by default, and only keep metrics that match a regex or have a specific label and value.

Metric collectors process metrics through their own metric filters and transforms prior to forwarding them to any defined exporters. The exporters then apply their own filters and transforms before sending the metrics to the destination.

# Defines a prometheus format endpoint scraper.
[prometheus-scrape]
  # Database to store metrics in.
  database = 'Metrics'
  # Defines a static scrape target.
  static-scrape-target = []
  # Scrape interval in seconds.
  scrape-interval = 10
  # Scrape timeout in seconds.
  scrape-timeout = 5
  # Disable metrics forwarding to endpoints.
  disable-metrics-forwarding = false
  # Disable discovery of kubernetes pod targets.
  disable-discovery = false
  # Regexes of metrics to drop.
  drop-metrics = []
  # Regexes of metrics to keep.
  keep-metrics = []
  # Regexes of metrics to keep if they have the given label and value.
  keep-metrics-with-label-value = []
  # List of exporter names to forward metrics to.
  exporters = [
    'to-local-otlp',
    'to-remote-otlp'
  ]

# Optional configuration for exporting telemetry outside of adx-mon in parallel with sending to ADX.
# Exporters are declared here and referenced by name in each collection source.
[exporters]
  # Configuration for exporting metrics to an OTLP/HTTP endpoint.
  [[exporters.otlp-metric-export]]
    # Name of the exporter.
    name = 'to-local-otlp'
    # OTLP/HTTP endpoint to send metrics to.
    destination = 'http://localhost:4318/v1/metrics'
    # Default to dropping all metrics.  Only metrics matching a keep rule will be kept.
    default-drop-metrics = true
    # Regexes of metrics to drop.
    drop-metrics = []
    # Regexes of metrics to keep.
    keep-metrics = [
      '^kube_pod_ips$'
    ]
    # Regexes of metrics to keep if they have the given label and value.
    keep-metrics-with-label-value = []

    # Key/value pairs of labels to add to all metrics.
    [exporters.otlp-metric-export.add-labels]
      forwarded_to = 'otlp'

    # Labels to drop if they match a metrics regex in the format <metrics regex>=<label name>.
    [exporters.otlp-metric-export.drop-labels]
      '^kube_pod_ips$' = '^ip_family'

    # Key/value pairs of resource attributes to add to all metrics.
    [exporters.otlp-metric-export.add-resource-attributes]
      destination_namespace = 'prod-metrics'

  [[exporters.otlp-metric-export]]
    # Name of the exporter.
    name = 'to-remote-otlp'
    # OTLP/HTTP endpoint to send metrics to.
    destination = 'https://metrics.contoso.org/v1/metrics'
    # Default to dropping all metrics.  Only metrics matching a keep rule will be kept.
    default-drop-metrics = true
    # Regexes of metrics to drop.
    drop-metrics = []
    # Regexes of metrics to keep.
    keep-metrics = [
      '^service_hit_count$',
      '^service_latency$'
    ]
    # Regexes of metrics to keep if they have the given label and value.
    keep-metrics-with-label-value = []

    # Key/value pairs of labels to add to all metrics.
    [exporters.otlp-metric-export.add-labels]
      forwarded_to = 'otlp'

    # Labels to drop if they match a metrics regex in the format <metrics regex>=<label name>.
    [exporters.otlp-metric-export.drop-labels]
      '^service_hit_count$' = '^origin_ip$'

    # Key/value pairs of resource attributes to add to all metrics.
    [exporters.otlp-metric-export.add-resource-attributes]
      destination_namespace = 'primary-metrics'

Log Exporters

Logs currently support exporting to fluent-forward tcp or unix domain socket endpoints with fluent-forward-log-export. This exporter forwards logs to the remote endpoint with a tag based on the value of the attribute tag-attribute within the log.

As an example, if 'tag-attribute' is set to 'fluent-output-tag', logs with an attribute of fluent-output-tag -> service-logs will be emitted with the tag service-logs. If the attribute is not present, the log will not be emitted by this exporter.

# Defines a host log scraper.
[[host-log]]
  # Disable discovery of Kubernetes pod targets. Only one HostLog configuration can use Kubernetes discovery.
  disable-kube-discovery = false
  # Defines a tail file target.
  file-target = []
  # Defines a static Kubernetes pod target to scrape. These are pods managed by the Kubelet and not discoverable via the apiserver.
  static-pod-target = []
  # Defines a journal target to scrape.
  journal-target = []
  # Defines a kernel target to scrape.
  kernel-target = []
  # Defines a list of transforms to apply to log lines.
  transforms = []
  # List of exporter names to forward logs to.
  exporters = [
    'fluentd-tcp',
    'fluentd-unix'
  ]

# Optional configuration for exporting telemetry outside of adx-mon in parallel with sending to ADX.
# Exporters are declared here and referenced by name in each collection source.
[exporters]
  # Configuration for exporting logs to a Fluentd/Fluent Bit endpoint.
  [[exporters.fluent-forward-log-export]]
    # Name of the exporter.
    name = 'fluentd-tcp'
    # Fluentd/Fluent Bit endpoint to send logs to. Must be in the form tcp://<host>:<port> or unix:///path/to/socket.
    destination = 'tcp://localhost:24224'
    # Attribute key to use as the tag for the log. If the attribute is not set, the log will be ignored by this exporter.
    tag-attribute = 'fluent-output-tag-tcp'

  [[exporters.fluent-forward-log-export]]
    # Name of the exporter.
    name = 'fluentd-unix'
    # Fluentd/Fluent Bit endpoint to send logs to. Must be in the form tcp://<host>:<port> or unix:///path/to/socket.
    destination = 'unix:///var/run/fluent.sock'
    # Attribute key to use as the tag for the log. If the attribute is not set, the log will be ignored by this exporter.
    tag-attribute = 'fluent-output-tag-unix'