Monitor a Linux VM

receivers:
  host_metrics:
    collection_interval: 60s
    scrapers:
      cpu:
        metrics:
          system.cpu.time: { enabled: false }
          system.cpu.utilization: { enabled: true }
          system.cpu.logical.count: { enabled: true }
      load:
        # Load divided by the CPU count: 1.0 is saturated on any VM size.
        cpu_average: true
      memory:
        metrics:
          system.memory.utilization: { enabled: true }
          system.memory.limit: { enabled: true }
          system.linux.memory.available: { enabled: true }
      paging:
        metrics:
          system.paging.utilization: { enabled: true }
      disk:
        # Whole disks only.
        include:
          devices:
            - '^(sd[a-z]+|vd[a-z]+|xvd[a-z]+|nvme[0-9]+n[0-9]+)$'
          match_type: regexp
      filesystem:
        # Disk-backed filesystems only: skips tmpfs, snaps, container layers,
        # the EFI partition, and read-only images.
        include_fs_types:
          fs_types: [ext4, xfs, btrfs]
          match_type: strict
        metrics:
          system.filesystem.utilization: { enabled: true }
      network:
        # Real NICs only.
        include:
          interfaces:
            - '^(eth[0-9]+|en[a-z0-9]+)$'
          match_type: regexp
      processes: {}
      system:
        metrics:
          system.uptime: { enabled: true }

  # Out-of-memory kills, disk errors, failed systemd units.
  journald:
    priority: warning
    matches:
      - _TRANSPORT: kernel
      - _PID: "1"
    start_at: end

processors:
  memory_limiter:
    check_interval: 1s
    limit_mib: 200
    spike_limit_mib: 50
  resource_detection:
    detectors: [env, system]
    system:
      hostname_sources: [os]
      resource_attributes:
        host.id: { enabled: true }
  resource/sherlock:
    attributes:
      - { key: service.name, value: vm-health, action: insert }
      # prod or dev; see Environments.
      - { key: env, value: prod, action: insert }
  # The unit a systemd message is about, a severity, and the message as the body.
  transform/journald:
    error_mode: ignore
    log_statements:
      - context: log
        statements:
          - set(log.attributes["systemd.unit"], log.body["UNIT"])
            where log.body["UNIT"] != nil
          - set(log.severity_text, "ERROR")
            where IsMatch(log.body["PRIORITY"], "^[0-3]$")
          - set(log.severity_number, SEVERITY_NUMBER_ERROR)
            where IsMatch(log.body["PRIORITY"], "^[0-3]$")
          - set(log.severity_text, "WARN") where log.body["PRIORITY"] == "4"
          - set(log.severity_number, SEVERITY_NUMBER_WARN)
            where log.body["PRIORITY"] == "4"
          - set(log.body, log.body["MESSAGE"])
            where log.body["MESSAGE"] != nil
  # Hold data up to 2 s so a burst of journal lines goes out as one request.
  batch:
    timeout: 2s

exporters:
  otlp_http:
    endpoint: ${env:SHERLOCK_ENDPOINT}
    headers:
      Authorization: "Bearer ${env:SHERLOCK_TOKEN}"
    # Keep retrying through an ingest outage of up to 15 minutes (default 5).
    retry_on_failure:
      max_elapsed_time: 15m

service:
  telemetry:
    metrics: { level: none }
  pipelines:
    metrics:
      receivers: [host_metrics]
      processors:
        - memory_limiter
        - resource_detection
        - resource/sherlock
        - batch
      exporters: [otlp_http]
    logs:
      receivers: [journald]
      processors:
        - memory_limiter
        - transform/journald
        - resource_detection
        - resource/sherlock
        - batch
      exporters: [otlp_http]