Collector Config

Collector is configured with a TOML-formatted file. In Kubernetes deployments, this is typically within a ConfigMap mounted into the collector pod. A default config can be generated by running ./collector config.

Global Config

This is the top level configuration for the collector. The only required fields are Endpoint and StorageDir.

# Ingestor URL to send collected telemetry.
endpoint = 'https://ingestor.adx-mon.svc.cluster.local'
# Path to kubernetes client config
kube-config = '.kube/config'
# Skip TLS verification.
insecure-skip-verify = true
# Address to listen on for endpoints.
listen-addr = ':8080'
# Region is a location identifier.
region = 'eastus'
# Optional path to the TLS key file.
tls-key-file = '/etc/certs/collector.key'
# Optional path to the TLS cert bundle file.
tls-cert-file = '/etc/certs/collector.pem'
# Maximum number of connections to accept.
max-connections = 100
# Maximum number of samples to send in a single batch.
max-batch-size = 1000
# Max segment agent in seconds.
max-segment-age-seconds = 30
# Maximum segment size in bytes.
max-segment-size = 52428800
# Maximum allowed size in bytes of all segments on disk.
max-disk-usage = 53687091200
# Interval to flush the WAL. (default 100)
wal-flush-interval-ms = 100
# Storage directory for the WAL and log cursors.
storage-dir = '/var/lib/adx-mon'
# Enable pprof endpoints.
enable-pprof = true
# Default to dropping all metrics.  Only metrics matching a keep rule will be kept.
default-drop-metrics = false
# Global Regexes of metrics to drop.
drop-metrics = [
  '^kube_pod_ips$',
  'etcd_grpc.*'
]
# Global Regexes of metrics to keep.
keep-metrics = [
  'nginx.*'
]
# Attributes lifted from the Body field and added to Attributes.
lift-attributes = [
  'host'
]

# Global Key/value pairs of labels to add to all metrics.
[add-labels]
  collectedBy = 'collector'

# Global labels to drop if they match a metrics regex in the format <metrics regex>=<label name>. These are dropped from all metrics collected by this agent
[drop-labels]
  '^nginx_connections_accepted' = '^pid$'

# Global Regexes of metrics to keep if they have the given label and value. These are kept from all metrics collected by this agent
[[keep-metrics-with-label-value]]
  # The regex to match the label value against.  If the label value matches, the metric will be kept.
  label-regex = 'owner'
  # The regex to match the label value against.  If the label value matches, the metric will be kept.
  value-regex = 'platform'

[[keep-metrics-with-label-value]]
  # The regex to match the label value against.  If the label value matches, the metric will be kept.
  label-regex = 'type'
  # The regex to match the label value against.  If the label value matches, the metric will be kept.
  value-regex = 'frontend|backend'

# Global labels to lift from the metric to top level columns
[[lift-labels]]
  # The name of the label to lift.
  name = 'Host'
  # The name of the column to lift the label to.
  column = ''

[[lift-labels]]
  # The name of the label to lift.
  name = 'cluster_name'
  # The name of the column to lift the label to.
  column = 'Cluster'

# Key/value pairs of attributes to add to all logs.
[add-attributes]
  cluster = 'cluster1'
  geo = 'eu'

# Optional configuration for exporting telemetry outside of adx-mon in parallel with sending to ADX.
# Exporters are declared here and referenced by name in each collection source.
[exporters]
  # Configuration for exporting metrics to an OTLP/HTTP endpoint.
  [[exporters.otlp-metric-export]]
    # Name of the exporter.
    name = 'to-otlp'
    # OTLP/HTTP endpoint to send metrics to.
    destination = 'http://localhost:4318/v1/metrics'
    # Default to dropping all metrics.  Only metrics matching a keep rule will be kept.
    default-drop-metrics = true
    # Regexes of metrics to drop.
    drop-metrics = []
    # Regexes of metrics to keep.
    keep-metrics = [
      '^kube_pod_ips$'
    ]
    # Regexes of metrics to keep if they have the given label and value.
    keep-metrics-with-label-value = []

    # Key/value pairs of labels to add to all metrics.
    [exporters.otlp-metric-export.add-labels]
      forwarded_to = 'otlp'

    # Labels to drop if they match a metrics regex in the format <metrics regex>=<label name>.
    [exporters.otlp-metric-export.drop-labels]
      '^kube_pod_ips$' = '^ip_family'

    # Key/value pairs of resource attributes to add to all metrics.
    [exporters.otlp-metric-export.add-resource-attributes]
      destination_namespace = 'prod-metrics'

Prometheus Scrape

Prometheus scrape discovers pods with the adx-mon/scrape annotation as well as any defined static scrape targets. It ships any metrics to the defined ADX database.

# Defines a prometheus format endpoint scraper.
[prometheus-scrape]
  # Database to store metrics in.
  database = 'Metrics'
  # Scrape interval in seconds.
  scrape-interval = 10
  # Scrape timeout in seconds.
  scrape-timeout = 5
  # Disable metrics forwarding to endpoints.
  disable-metrics-forwarding = false
  # Disable discovery of kubernetes pod targets.
  disable-discovery = false
  # Regexes of metrics to drop.
  drop-metrics = [
    '^kube_pod_ips$',
    'etcd_grpc.*'
  ]
  # Regexes of metrics to keep.
  keep-metrics = [
    'nginx.*'
  ]
  # List of exporter names to forward metrics to.
  exporters = []

  # Defines a static scrape target.
  [[prometheus-scrape.static-scrape-target]]
    # The regex to match the host name against.  If the hostname matches, the URL will be scraped.
    host-regex = '.*'
    # The URL to scrape.
    url = 'http://localhost:9090/metrics'
    # The namespace label to add for metrics scraped at this URL.
    namespace = 'monitoring'
    # The pod label to add for metrics scraped at this URL.
    pod = 'host-monitor'
    # The container label to add for metrics scraped at this URL.
    container = 'host-monitor'

  # Regexes of metrics to keep if they have the given label and value.
  [[prometheus-scrape.keep-metrics-with-label-value]]
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    label-regex = 'owner'
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    value-regex = 'platform'

  [[prometheus-scrape.keep-metrics-with-label-value]]
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    label-regex = 'type'
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    value-regex = 'frontend|backend'

Prometheus Remote Write

Prometheus remote write accepts metrics from Prometheus remote write protocol. It ships metrics to the defined ADX database.

# Defines a prometheus remote write endpoint.
[[prometheus-remote-write]]
  # Database to store metrics in.
  database = 'Metrics'
  # The path to listen on for prometheus remote write requests.  Defaults to /receive.
  path = '/receive'
  # Regexes of metrics to drop.
  drop-metrics = [
    '^kube_pod_ips$',
    'etcd_grpc.*'
  ]
  # Regexes of metrics to keep.
  keep-metrics = [
    'nginx.*'
  ]
  # List of exporter names to forward metrics to.
  exporters = []

  # Key/value pairs of labels to add to all metrics.
  [prometheus-remote-write.add-labels]
    cluster = 'cluster1'

  # Labels to drop if they match a metrics regex in the format <metrics regex>=<label name>.
  [prometheus-remote-write.drop-labels]
    '^nginx_connections_accepted' = '^pid$'

  # Regexes of metrics to keep if they have the given label and value.
  [[prometheus-remote-write.keep-metrics-with-label-value]]
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    label-regex = 'owner'
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    value-regex = 'platform'

  [[prometheus-remote-write.keep-metrics-with-label-value]]
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    label-regex = 'type'
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    value-regex = 'frontend|backend'

Otel Log

The Otel log endpoint accepts OTLP/HTTP logs from an OpenTelemetry sender. By default, this listens under the path /v1/logs.

# Defines an OpenTelemetry log endpoint. Accepts OTLP/HTTP.
[otel-log]
  # Attributes lifted from the Body and added to Attributes.
  lift-attributes = [
    'host'
  ]

  # Key/value pairs of attributes to add to all logs.
  [otel-log.add-attributes]
    cluster = 'cluster1'
    geo = 'eu'

Otel Metrics

The Otel metrics endpoint accepts OTLP/HTTP and/or OTLP/gRPC metrics from an OpenTelemetry sender.

# Defines an OpenTelemetry metric endpoint. Accepts OTLP/HTTP and/or OTLP/gRPC.
[[otel-metric]]
  # Database to store metrics in.
  database = 'Metrics'
  # The path to listen on for OTLP/HTTP requests.
  path = '/v1/otlpmetrics'
  # The port to listen on for OTLP/gRPC requests.
  grpc-port = 4317
  # Regexes of metrics to drop.
  drop-metrics = [
    '^kube_pod_ips$',
    'etcd_grpc.*'
  ]
  # Regexes of metrics to keep.
  keep-metrics = [
    'nginx.*'
  ]
  # List of exporter names to forward metrics to.
  exporters = []

  # Key/value pairs of labels to add to all metrics.
  [otel-metric.add-labels]
    cluster = 'cluster1'

  # Labels to drop if they match a metrics regex in the format <metrics regex>=<label name>.  These are dropped from all metrics collected by this agent
  [otel-metric.drop-labels]
    '^nginx_connections_accepted' = '^pid$'

  # Regexes of metrics to keep if they have the given label and value.
  [[otel-metric.keep-metrics-with-label-value]]
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    label-regex = 'owner'
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    value-regex = 'platform'

  [[otel-metric.keep-metrics-with-label-value]]
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    label-regex = 'type'
    # The regex to match the label value against.  If the label value matches, the metric will be kept.
    value-regex = 'frontend|backend'

Host Log

The host log config configures file and journald log collection. By default, Kubernetes pods with adx-mon/log-destination annotation will have their logs scraped and sent to the appropriate destinations.

# Defines a host log scraper.
[[host-log]]
  # Disable discovery of Kubernetes pod targets. Only one HostLog configuration can use Kubernetes discovery.
  disable-kube-discovery = false
  # Defines a list of transforms to apply to log lines.
  transforms = []

  # Key/value pairs of attributes to add to all logs.
  [host-log.add-attributes]
    cluster = 'cluster1'
    geo = 'eu'

  # Defines a tail file target.
  [[host-log.file-target]]
    # The path to the file to tail.
    file-path = '/var/log/nginx/access.log'
    # The type of log being output. This defines how timestamps and log messages are extracted from structured log types like docker json files. Options are: docker, plain.
    log-type = 'plain'
    # Database to store logs in.
    database = 'Logs'
    # Table to store logs in.
    table = 'NginxAccess'
    # Parsers to apply sequentially to the log line.
    parsers = []

  [[host-log.file-target]]
    # The path to the file to tail.
    file-path = '/var/log/myservice/service.log'
    # The type of log being output. This defines how timestamps and log messages are extracted from structured log types like docker json files. Options are: docker, plain.
    log-type = 'plain'
    # Database to store logs in.
    database = 'Logs'
    # Table to store logs in.
    table = 'NginxAccess'
    # Parsers to apply sequentially to the log line.
    parsers = [
      'json'
    ]

  # Defines a journal target to scrape.
  [[host-log.journal-target]]
    # Matches for the journal reader based on journalctl MATCHES. To select a systemd unit, use the field _SYSTEMD_UNIT. (e.g. '_SYSTEMD_UNIT=avahi-daemon.service' for selecting logs from the avahi-daemon service.)
    matches = [
      '_SYSTEMD_UNIT=docker.service',
      '_TRANSPORT=journal'
    ]
    # Database to store logs in.
    database = 'Logs'
    # Table to store logs in.
    table = 'Docker'
    # Parsers to apply sequentially to the log line.
    parsers = []

Exporters

Exporters are used to send telemetry to external systems in parallel with data sent to Azure Data Explorer. The collector currently supports sending metrics to OpenTelemetry OTLP/HTTP endpoints. Exporters are defined under the top level configuration key exportersunder the exporter type. They are referenced by name in each metric collector.

Metric collectors process metrics through their own metric filters and transforms prior to forwarding them to any defined exporters. The exporters then apply their own filters and transforms before sending the metrics to the destination.

# Defines a prometheus format endpoint scraper.
[prometheus-scrape]
  # Database to store metrics in.
  database = 'Metrics'
  # Defines a static scrape target.
  static-scrape-target = []
  # Scrape interval in seconds.
  scrape-interval = 10
  # Scrape timeout in seconds.
  scrape-timeout = 5
  # Disable metrics forwarding to endpoints.
  disable-metrics-forwarding = false
  # Disable discovery of kubernetes pod targets.
  disable-discovery = false
  # Regexes of metrics to drop.
  drop-metrics = []
  # Regexes of metrics to keep.
  keep-metrics = []
  # Regexes of metrics to keep if they have the given label and value.
  keep-metrics-with-label-value = []
  # List of exporter names to forward metrics to.
  exporters = [
    'to-local-otlp',
    'to-remote-otlp'
  ]

# Optional configuration for exporting telemetry outside of adx-mon in parallel with sending to ADX.
# Exporters are declared here and referenced by name in each collection source.
[exporters]
  # Configuration for exporting metrics to an OTLP/HTTP endpoint.
  [[exporters.otlp-metric-export]]
    # Name of the exporter.
    name = 'to-local-otlp'
    # OTLP/HTTP endpoint to send metrics to.
    destination = 'http://localhost:4318/v1/metrics'
    # Default to dropping all metrics.  Only metrics matching a keep rule will be kept.
    default-drop-metrics = true
    # Regexes of metrics to drop.
    drop-metrics = []
    # Regexes of metrics to keep.
    keep-metrics = [
      '^kube_pod_ips$'
    ]
    # Regexes of metrics to keep if they have the given label and value.
    keep-metrics-with-label-value = []

    # Key/value pairs of labels to add to all metrics.
    [exporters.otlp-metric-export.add-labels]
      forwarded_to = 'otlp'

    # Labels to drop if they match a metrics regex in the format <metrics regex>=<label name>.
    [exporters.otlp-metric-export.drop-labels]
      '^kube_pod_ips$' = '^ip_family'

    # Key/value pairs of resource attributes to add to all metrics.
    [exporters.otlp-metric-export.add-resource-attributes]
      destination_namespace = 'prod-metrics'

  [[exporters.otlp-metric-export]]
    # Name of the exporter.
    name = 'to-remote-otlp'
    # OTLP/HTTP endpoint to send metrics to.
    destination = 'https://metrics.contoso.org/v1/metrics'
    # Default to dropping all metrics.  Only metrics matching a keep rule will be kept.
    default-drop-metrics = true
    # Regexes of metrics to drop.
    drop-metrics = []
    # Regexes of metrics to keep.
    keep-metrics = [
      '^service_hit_count$',
      '^service_latency$'
    ]
    # Regexes of metrics to keep if they have the given label and value.
    keep-metrics-with-label-value = []

    # Key/value pairs of labels to add to all metrics.
    [exporters.otlp-metric-export.add-labels]
      forwarded_to = 'otlp'

    # Labels to drop if they match a metrics regex in the format <metrics regex>=<label name>.
    [exporters.otlp-metric-export.drop-labels]
      '^service_hit_count$' = '^origin_ip$'

    # Key/value pairs of resource attributes to add to all metrics.
    [exporters.otlp-metric-export.add-resource-attributes]
      destination_namespace = 'primary-metrics'