Grafana and Loki

# provisioning/datasources/datasources.yaml
apiVersion: 1
datasources:
  - name: Prometheus
    uid: prometheus                    # fixed uid so dashboards and alert rules can reference it without an id lookup
    type: prometheus
    access: proxy                      # Grafana's backend makes the request; the browser never reaches the datasource
    url: http://prometheus.example.com:9090
    isDefault: true
    editable: false
    jsonData:
      httpMethod: POST
      timeInterval: 15s                # scrape interval, drives $__rate_interval
      exemplarTraceIdDestinations:
        - name: trace_id
          datasourceUid: tempo
  - name: Loki
    uid: loki
    type: loki
    access: proxy
    url: http://loki.example.com:3100
    jsonData:
      maxLines: 1000
      derivedFields:                   # turn a trace id in a log line into a link
        - name: TraceID
          matcherRegex: 'trace_id=(\w+)'
          url: '$${__value.raw}'       # $$ escapes Grafana's own interpolation in provisioning files
          datasourceUid: tempo
    secureJsonData:
      httpHeaderValue1: "$LOKI_TENANT" # environment variables expand; keep real values out of the file
    jsonData:
      httpHeaderName1: X-Scope-OrgID
deleteDatasources:
  - name: Old Prometheus
    orgId: 1

# provisioning/dashboards/default.yaml
apiVersion: 1
providers:
  - name: repo
    orgId: 1
    type: file
    updateIntervalSeconds: 30
    allowUiUpdates: false              # true lets people save from the UI, and the file still wins at the next change
    disableDeletion: false             # false: removing the file removes the dashboard
    options:
      path: /var/lib/grafana/dashboards
      foldersFromFilesStructure: true  # subdirectory name becomes the folder

# provisioning/alerting/contact-points.yaml
apiVersion: 1
contactPoints:
  - orgId: 1
    name: oncall-slack
    receivers:
      - uid: oncall-slack
        type: slack
        settings:
          url: "$SLACK_WEBHOOK_URL"    # expanded from the environment at load
          recipient: "#oncall"
          title: '{{ template "default.title" . }}'
        disableResolveMessage: false
policies:
  - orgId: 1
    receiver: oncall-slack             # root policy: everything not matched below
    group_by: [alertname, namespace]
    group_wait: 30s
    group_interval: 5m
    repeat_interval: 4h
    routes:
      - receiver: oncall-slack
        object_matchers:
          - [severity, "=", critical]
        repeat_interval: 1h
        continue: false

{
  "uid": "my-app",
  "title": "my-app",
  "tags": ["team-platform"],
  "editable": false,
  "schemaVersion": 41,
  "time": { "from": "now-6h", "to": "now" },
  "refresh": "1m",
  "templating": { "list": [] },
  "panels": [
    {
      "type": "timeseries",
      "title": "Requests per second",
      "gridPos": { "x": 0, "y": 0, "w": 12, "h": 8 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "targets": [
        {
          "refId": "A",
          "expr": "sum by (status) (rate(http_requests_total{job=\"my-app\", namespace=\"$namespace\"}[$__rate_interval]))",
          "legendFormat": "{{status}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "reqps",
          "min": 0,
          "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 500 } ] }
        },
        "overrides": [
          { "matcher": { "id": "byRegexp", "options": "5.." }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] }
        ]
      },
      "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max", "lastNotNull"] }, "tooltip": { "mode": "multi", "sort": "desc" } }
    }
  ]
}

{
  "name": "namespace",
  "type": "query",
  "datasource": { "type": "prometheus", "uid": "prometheus" },
  "query": { "query": "label_values(kube_namespace_status_phase, namespace)", "refId": "ns" },
  "refresh": 2,
  "regex": "/^(?!kube-).*/",
  "sort": 1,
  "multi": true,
  "includeAll": true,
  "allValue": ".+",
  "current": { "text": "All", "value": "$__all" }
}

apiVersion: 1
groups:
  - orgId: 1
    name: my-app
    folder: platform
    interval: 1m
    rules:
      - uid: my-app-error-ratio
        title: my-app 5xx ratio above 5%
        condition: C
        for: 10m
        noDataState: NoData          # a query returning nothing is not an outage; alert on absent() separately
        execErrState: Error
        labels: { severity: critical, team: platform }
        annotations:
          summary: "{{ $labels.namespace }}/my-app 5xx ratio is {{ $values.B | printf \"%.1f\" }}%"
          runbook_url: https://wiki.example.com/runbooks/my-app-5xx
        data:
          - refId: A
            relativeTimeRange: { from: 600, to: 0 }
            datasourceUid: prometheus
            model:
              expr: 100 * sum by (namespace) (rate(http_requests_total{job="my-app",status=~"5.."}[5m])) / sum by (namespace) (rate(http_requests_total{job="my-app"}[5m]))
              instant: true
              refId: A
          - refId: B
            datasourceUid: __expr__
            model: { type: reduce, expression: A, reducer: last, refId: B }
          - refId: C
            datasourceUid: __expr__
            model: { type: threshold, expression: B, refId: C, conditions: [ { evaluator: { type: gt, params: [5] } } ] }

# Error ratio as a percentage, safe when the denominator is zero
100 * sum(rate(http_requests_total{job="my-app",status=~"5.."}[$__rate_interval])) / clamp_min(sum(rate(http_requests_total{job="my-app"}[$__rate_interval])), 1)

# p99 latency from a histogram; a heatmap panel takes the same query without histogram_quantile and format=heatmap
histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket{job="my-app"}[$__rate_interval])))

# Stat panel: total over the dashboard time range, one instant value
sum(increase(jobs_completed_total{job="my-app"}[$__range]))

# Table: current value per pod with restart count, format=table, instant=true, then Transform > Organize fields
kube_pod_container_status_restarts_total{namespace="$namespace"}

# Legend with variables and a fixed unit: legendFormat "{{pod}} ({{container}})", unit "bytes"
container_memory_working_set_bytes{namespace="$namespace", container!=""}

# Deployment markers: annotation query on a counter that increments on rollout
changes(kube_deployment_status_observed_generation{namespace="$namespace"}[1m]) > 0

G=http://grafana.example.com:3000
H="Authorization: Bearer $GRAFANA_TOKEN"
curl -fsS "$G/api/health"                                          # no auth: {"database":"ok","version":...}
curl -fsS -H "$H" "$G/api/org"                                     # confirms the token works and which org it is in
curl -fsS -H "$H" "$G/api/search?type=dash-db&tag=team-platform" | jq -r '.[] | "\(.uid)\t\(.folderTitle)/\(.title)"'
curl -fsS -H "$H" "$G/api/dashboards/uid/my-app" | jq '.dashboard' > my-app.json     # .meta has folder, version, provisioned flag
curl -fsS -H "$H" "$G/api/dashboards/uid/my-app/versions" | jq '.versions[] | {version, created, createdBy, message}'
curl -fsS -H "$H" "$G/api/folders" | jq '.[] | {uid, title}'
curl -fsS -X POST -H "$H" -H 'Content-Type: application/json' "$G/api/folders" -d '{"uid":"ops","title":"Ops"}'
# Import: id must be null, overwrite replaces a dashboard with the same uid; provisioned dashboards return 400
jq '{dashboard: (. + {id: null}), folderUid: "ops", overwrite: true, message: "from CI"}' my-app.json \
  | curl -fsS -X POST -H "$H" -H 'Content-Type: application/json' "$G/api/dashboards/db" -d @-
curl -fsS -X DELETE -H "$H" "$G/api/dashboards/uid/old-dashboard"   # permanent; export first
curl -fsS -H "$H" "$G/api/datasources" | jq '.[] | {name, type, uid, url}'
curl -fsS -H "$H" "$G/api/datasources/uid/prometheus/health"
# Run a query through Grafana (uses the datasource's credentials, useful when the datasource is not reachable directly)
curl -fsS -X POST -H "$H" -H 'Content-Type: application/json' "$G/api/ds/query" -d '{"queries":[{"refId":"A","datasource":{"uid":"prometheus"},"expr":"up","instant":true}],"from":"now-5m","to":"now"}' | jq '.results.A.frames[0].data'
# Annotation on a dashboard, for deploy markers
curl -fsS -X POST -H "$H" -H 'Content-Type: application/json' "$G/api/annotations" -d "{\"dashboardUID\":\"my-app\",\"time\":$(date +%s000),\"tags\":[\"deploy\"],\"text\":\"my-app 1.4.2\"}"
# Alerting provisioning API: rules, contact points and policies as the same YAML the files use
curl -fsS -H "$H" "$G/api/v1/provisioning/alert-rules" | jq '.[] | {uid, title, folderUID}'
curl -fsS -H "$H" "$G/api/v1/provisioning/contact-points/export?format=yaml"
curl -fsS -H "$H" "$G/api/v1/provisioning/policies/export?format=yaml"
# Service account and token (admin)
curl -fsS -X POST -H "$H" -H 'Content-Type: application/json' "$G/api/serviceaccounts" -d '{"name":"ci","role":"Editor"}'
curl -fsS -X POST -H "$H" -H 'Content-Type: application/json' "$G/api/serviceaccounts/12/tokens" -d '{"name":"ci-token"}'   # prints the key once

# /etc/loki/config.yaml, single node with filesystem storage
auth_enabled: false                 # single tenant; multi-tenant deployments require X-Scope-OrgID on every request
server:
  http_listen_port: 3100
  grpc_listen_port: 9096
common:
  path_prefix: /var/lib/loki
  replication_factor: 1
  ring:
    kvstore: { store: inmemory }
  storage:
    filesystem:
      chunks_directory: /var/lib/loki/chunks
      rules_directory: /var/lib/loki/rules
schema_config:
  configs:
    - from: "2024-01-01"
      store: tsdb
      object_store: filesystem
      schema: v13
      index: { prefix: index_, period: 24h }
limits_config:
  retention_period: 744h            # 31 days; needs the compactor block below
  ingestion_rate_mb: 8
  ingestion_burst_size_mb: 16
  per_stream_rate_limit: 5MB
  per_stream_rate_limit_burst: 20MB
  max_query_series: 1000
  max_query_length: 0h              # 0 removes the 30-day default cap
  reject_old_samples: true
  reject_old_samples_max_age: 168h
  allow_structured_metadata: true
  volume_enabled: true
compactor:
  working_directory: /var/lib/loki/compactor
  retention_enabled: true
  delete_request_store: filesystem
ruler:
  alertmanager_url: http://alertmanager.example.com:9093

# /etc/promtail/config.yaml
server: { http_listen_port: 9080 }
positions: { filename: /var/lib/promtail/positions.yaml }   # where each file was left off; losing it re-reads everything
clients:
  - url: http://loki.example.com:3100/loki/api/v1/push
    external_labels: { host: web-1 }
scrape_configs:
  - job_name: journal
    journal:
      max_age: 12h
      labels: { job: systemd-journal }
    relabel_configs:
      - source_labels: ['__journal__systemd_unit']
        target_label: unit
  - job_name: my-app
    static_configs:
      - targets: [localhost]
        labels: { job: my-app, __path__: /var/log/my-app/*.log }
    pipeline_stages:
      - json:
          expressions: { level: level, ts: timestamp }
      - timestamp: { source: ts, format: RFC3339Nano }
      - labels: { level: "" }      # promote level to a label; bounded set, safe

// /etc/alloy/config.alloy
loki.source.journal "journal" {
  max_age       = "12h"
  labels        = { job = "systemd-journal" }
  relabel_rules = loki.relabel.journal.rules
  forward_to    = [loki.write.default.receiver]
}

loki.relabel "journal" {
  forward_to = []
  rule {
    source_labels = ["__journal__systemd_unit"]
    target_label  = "unit"
  }
}

local.file_match "my_app" {
  path_targets = [{ __path__ = "/var/log/my-app/*.log", job = "my-app" }]
}

loki.source.file "my_app" {
  targets    = local.file_match.my_app.targets
  forward_to = [loki.process.my_app.receiver]
}

loki.process "my_app" {
  stage.json {
    expressions = { level = "level", ts = "timestamp" }
  }
  stage.timestamp {
    source = "ts"
    format = "RFC3339Nano"
  }
  stage.labels {
    values = { level = "" }
  }
  stage.structured_metadata {
    values = { trace_id = "" }        // high cardinality: metadata, not a label
  }
  forward_to = [loki.write.default.receiver]
}

loki.write "default" {
  endpoint {
    url = "http://loki.example.com:3100/loki/api/v1/push"
  }
  external_labels = { host = constants.hostname }
}

alloy fmt --verify /etc/alloy/config.alloy       # syntax and formatting; exit 1 on a diff
alloy run --server.http.listen-addr=127.0.0.1:12345 /etc/alloy/config.alloy   # UI at /  shows every component, its health and discovered targets
curl -fsS http://127.0.0.1:12345/-/ready
curl -fsS http://127.0.0.1:12345/metrics | grep -E '^loki_(source_file_read_bytes_total|write_(sent|dropped)_entries_total)'

export LOKI_ADDR=http://loki.example.com:3100
export LOKI_ORG_ID=my-tenant                       # only when auth_enabled: true
logcli labels                                       # label names in the default last hour
logcli labels app --since 24h
logcli series '{namespace="prod"}' --since 1h       # every stream
logcli series '{namespace="prod"}' --analyze-labels # per-label value counts: the high ones are your cardinality problem
logcli query --since 1h --limit 500 '{app="my-app"} |= "error"'                   # default: timestamp, labels, line
logcli query --since 1h -o raw '{app="my-app"} | json | line_format "{{.msg}}"'  # lines only
logcli query --since 1h -o jsonl '{app="my-app"}' | jq -r '.line'
logcli query --from="2026-09-23T22:00:00Z" --to="2026-09-24T02:00:00Z" --timezone=UTC --limit 0 --forward '{app="my-app"}' > incident.log   # --limit 0 removes the cap
logcli query --tail --delay-for 5 '{app="my-app"} |= "error"'      # follow, buffering 5 s for late lines
logcli query --stats --since 1h '{app="my-app"} |~ "timeout"' 2>&1 >/dev/null | grep -E 'Summary|TotalBytesProcessed|ExecTime'   # cost of a query
logcli instant-query 'sum by (app) (count_over_time({namespace="prod"} |= "error" [1h]))'
logcli query --since 6h --step 5m 'sum(rate({app="my-app"} |= "error" [5m]))' -o jsonl   # range metric query
logcli volume '{namespace="prod"}' --since 24h                                           # bytes per stream (volume_enabled: true)
logcli query --parallel-duration=15m --parallel-max-workers=4 --part-path-prefix=/var/tmp/export/my-app --merge-parts --from="2026-09-23T00:00:00Z" --to="2026-09-24T00:00:00Z" '{app="my-app"}'   # large exports in parallel parts

limits_config:
  retention_period: 744h
  retention_stream:
    - selector: '{namespace="dev"}'
      priority: 1
      period: 72h
    - selector: '{app="audit"}'
      priority: 2
      period: 8760h
compactor:
  retention_enabled: true
  delete_request_store: s3
  compaction_interval: 10m
  retention_delete_delay: 2h          # marked chunks are deleted this long after being marked
  retention_delete_worker_count: 150

curl -fsS -X POST -G "$LOKI_ADDR/loki/api/v1/delete" --data-urlencode 'query={app="my-app"} |= "password="' --data-urlencode "start=$(date -d '7 days ago' +%s)" --data-urlencode "end=$(date +%s)"   # deletes matching lines; irreversible once applied
curl -fsS "$LOKI_ADDR/loki/api/v1/delete" | jq .                                                                          # pending requests and status
curl -fsS -X DELETE "$LOKI_ADDR/loki/api/v1/delete?request_id=<id>"                                                       # cancel while still pending

# Every dashboard as a JSON file, one per uid, into ./dashboards
curl -fsS -H "Authorization: Bearer $GRAFANA_TOKEN" "$G/api/search?type=dash-db" | jq -r '.[].uid' | while read -r u; do curl -fsS -H "Authorization: Bearer $GRAFANA_TOKEN" "$G/api/dashboards/uid/$u" | jq '.dashboard' > "dashboards/$u.json"; done

# Dashboards nobody has opened in 90 days (needs [analytics] enabled; sort by usage in the UI otherwise)
curl -fsS -H "Authorization: Bearer $GRAFANA_TOKEN" "$G/api/search?type=dash-db&sort=views-asc&limit=20" | jq -r '.[] | .title'

# Datasources that fail their health check
curl -fsS -H "Authorization: Bearer $GRAFANA_TOKEN" "$G/api/datasources" | jq -r '.[].uid' | while read -r u; do s=$(curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $GRAFANA_TOKEN" "$G/api/datasources/uid/$u/health"); [[ $s == 200 ]] || echo "$u $s"; done

# Which dashboards use a given datasource uid
grep -l '"uid": "old-prometheus"' dashboards/*.json

# Which dashboards reference a metric
grep -l 'http_requests_total' dashboards/*.json

# Alert rules and their state, with the last error if any
curl -fsS -H "Authorization: Bearer $GRAFANA_TOKEN" "$G/api/prometheus/grafana/api/v1/rules" | jq -r '.data.groups[].rules[] | "\(.state)\t\(.name)\t\(.lastError // "")"'

# Silence one alert for two hours
curl -fsS -X POST -H "Authorization: Bearer $GRAFANA_TOKEN" -H 'Content-Type: application/json' "$G/api/alertmanager/grafana/api/v2/silences" -d "{\"matchers\":[{\"name\":\"alertname\",\"value\":\"my-app 5xx ratio above 5%\",\"isEqual\":true}],\"startsAt\":\"$(date -u +%FT%TZ)\",\"endsAt\":\"$(date -u -d '+2 hours' +%FT%TZ)\",\"createdBy\":\"ops\",\"comment\":\"deploy\"}"

# Grafana version and database status
curl -fsS "$G/api/health" | jq .

# Rows in the Grafana SQLite database that hold dashboards (on the server)
sqlite3 /var/lib/grafana/grafana.db 'select uid, title, updated from dashboard where is_folder = 0 order by updated desc limit 10'

# Loki: streams per app, highest first
logcli series '{namespace="prod"}' --since 1h -q | sed -E 's/.*app="([^"]*)".*/\1/' | sort | uniq -c | sort -rn | head

# Loki: bytes ingested per app over the last day
logcli instant-query 'topk(10, sum by (app) (bytes_over_time({namespace="prod"}[24h])))'

# Loki: apps that stopped logging in the last 30 minutes but logged in the hour before
logcli instant-query 'sum by (app) (count_over_time({namespace="prod"}[1h] offset 30m)) unless sum by (app) (count_over_time({namespace="prod"}[30m]))'

# Loki: error lines per minute for one app, as CSV
logcli query --since 6h --step 1m -o jsonl 'sum(count_over_time({app="my-app"} | json | level="error" [1m]))' | jq -r '.[] | .values[] | @csv' 2>/dev/null || true

# Loki: ingestion rate limits being hit, per tenant (Prometheus query against Loki's metrics)
curl -fsS "$LOKI_ADDR/metrics" | grep -E '^loki_discarded_samples_total' | grep -v ' 0$'

# Loki: ring health (every ingester should be ACTIVE)
curl -fsS "$LOKI_ADDR/ring" | grep -oE '(ACTIVE|LEAVING|PENDING|UNHEALTHY)' | sort | uniq -c

# Loki: flush in-memory chunks before a planned ingester shutdown
curl -fsS -X POST "$LOKI_ADDR/flush"

# Loki: compactor status and retention progress
curl -fsS "$LOKI_ADDR/compactor/ring"; curl -fsS "$LOKI_ADDR/metrics" | grep -E '^loki_compactor_(apply_retention_last_successful_run_timestamp_seconds|deleted_chunks_total)'

# Alloy: components in an unhealthy state
curl -fsS http://127.0.0.1:12345/api/v0/web/components | jq -r '.[] | select(.health.state != "healthy") | "\(.localID)\t\(.health.message)"'

# Alloy: lines dropped by the Loki writer (429s and 400s)
curl -fsS http://127.0.0.1:12345/metrics | grep -E '^loki_write_dropped_entries_total'

#!/usr/bin/env bash
# usage: GRAFANA_TOKEN=... grafana-export.sh http://grafana.example.com:3000 ./dashboards
set -euo pipefail
G=${1:?grafana url} out=${2:-dashboards}
H="Authorization: Bearer ${GRAFANA_TOKEN:?}"
mkdir -p "$out"
curl -fsS -H "$H" "$G/api/search?type=dash-db&limit=5000" \
  | jq -r '.[] | [.uid, (.folderTitle // "General")] | @tsv' \
  | while IFS=$'\t' read -r uid folder; do
      dir="$out/${folder// /-}"
      mkdir -p "$dir"
      curl -fsS -H "$H" "$G/api/dashboards/uid/$uid" | jq '.dashboard | .id = null' > "$dir/$uid.json"
      printf '%s/%s.json\n' "$dir" "$uid"
    done

#!/usr/bin/env bash
# usage: GRAFANA_TOKEN=... grafana-import.sh http://grafana.example.com:3000 ./dashboards
set -euo pipefail
G=${1:?grafana url} src=${2:-dashboards}
H="Authorization: Bearer ${GRAFANA_TOKEN:?}"
J='Content-Type: application/json'
find "$src" -mindepth 2 -name '*.json' | while read -r f; do
  folder=$(basename "$(dirname "$f")")
  fuid=$(curl -fsS -H "$H" "$G/api/folders" | jq -r --arg t "$folder" '.[] | select(.title == $t) | .uid')
  if [[ -z $fuid ]]; then
    fuid=$(curl -fsS -X POST -H "$H" -H "$J" "$G/api/folders" -d "$(jq -n --arg t "$folder" '{title: $t}')" | jq -r .uid)
  fi
  jq --arg fu "$fuid" '{dashboard: (. + {id: null}), folderUid: $fu, overwrite: true, message: "import script"}' "$f" \
    | curl -fsS -X POST -H "$H" -H "$J" "$G/api/dashboards/db" -d @- | jq -r '"\(.status)\t\(.uid)\t\(.url)"'
done

#!/usr/bin/env bash
# usage: LOKI_ADDR=... loki-cardinality.sh '{namespace="prod"}' 100
set -euo pipefail
sel=${1:-'{}'} threshold=${2:-100}
logcli series "$sel" --since 1h --analyze-labels -q \
  | awk -v t="$threshold" 'NR>1 && $2 ~ /^[0-9]+$/ { flag = ($2 > t) ? "HIGH" : ""; printf "%-40s %8s values %6s streams %s\n", $1, $2, $3, flag }' \
  | sort -k2 -nr

#!/usr/bin/env bash
set -euo pipefail
job=${1:?job label to check}
alloy=${ALLOY_ADDR:-http://127.0.0.1:12345}
: "${LOKI_ADDR:?}"
rc=0
curl -fsS --max-time 5 "$alloy/-/ready" >/dev/null || { echo "alloy not ready"; rc=1; }
curl -fsS --max-time 5 "$LOKI_ADDR/ready" >/dev/null || { echo "loki not ready"; rc=1; }
n=$(logcli instant-query -q "sum(count_over_time({job=\"$job\"}[5m]))" 2>/dev/null | jq -r '.[0].value[1] // "0"')
if [[ ${n%.*} -eq 0 ]]; then echo "no lines for job=$job in 5m"; rc=1; else echo "job=$job: $n lines in 5m"; fi
exit "$rc"