Prometheus

http_requests_total             # instant vector
http_requests_total[5m]         # range vector
rate(http_requests_total[5m])   # range in, instant out
sum(http_requests_total[5m])    # error: sum takes an instant vector

rate(http_requests_total[5m])       # per-second average over the window, reset-aware, extrapolated to the window edges
irate(http_requests_total[5m])      # per-second from the last two samples only: responsive, noisy
increase(http_requests_total[1h])   # rate × window; can return non-integers because of extrapolation

node_memory_MemAvailable_bytes
avg_over_time(node_memory_MemAvailable_bytes[1h])
delta(node_filesystem_avail_bytes[1h])                               # change over the window, gauges only
predict_linear(node_filesystem_avail_bytes[6h], 4*24*3600) < 0       # linear fit, 4 days ahead

histogram_quantile(0.99, sum by (le, service) (rate(http_request_duration_seconds_bucket[5m])))   # correct: rate, sum keeping le, then quantile
histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m]))                          # per-series quantile; usually not what you want

http_requests_total{job="api", status="500"}
http_requests_total{status=~"5.."}
http_requests_total{status!~"2..|3.."}
{__name__=~"http_.+", job="api"}

sum by (service, status) (rate(http_requests_total[5m]))      # keep only these labels
sum without (instance, pod) (rate(http_requests_total[5m]))   # drop these, keep the rest
topk(5, sum by (pod) (rate(container_cpu_usage_seconds_total[5m])))
count by (job) (up == 1)

# Attach the owning workload to per-pod CPU
sum by (pod) (rate(container_cpu_usage_seconds_total[5m]))
  * on (pod) group_left (owner_name)
kube_pod_owner

# Memory used as a fraction of the limit
sum by (pod) (container_memory_working_set_bytes)
  / on (pod) group_left
sum by (pod) (kube_pod_container_resource_limits{resource="memory"})

# Availability (SLI) over 30 days as a ratio
1 - (sum(increase(http_requests_total{status=~"5.."}[30d])) / sum(increase(http_requests_total[30d])))

# Error budget burn rate: 1.0 means spending exactly the budget for a 99.9% SLO
(sum(rate(http_requests_total{status=~"5.."}[1h])) / sum(rate(http_requests_total[1h]))) / (1 - 0.999)

# Multi-window burn-rate page: fast and slow windows must both burn
(sum(rate(http_requests_total{status=~"5.."}[1h])) / sum(rate(http_requests_total[1h])) > (14.4 * 0.001))
and
(sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) > (14.4 * 0.001))

# Fraction of requests under 300 ms (Apdex-style) from a classic histogram
sum(rate(http_request_duration_seconds_bucket{le="0.3"}[5m])) / sum(rate(http_request_duration_seconds_count[5m]))

# Week-over-week comparison of request rate
sum(rate(http_requests_total[5m])) / sum(rate(http_requests_total[5m] offset 1w))

# Value now versus one hour ago, as a percentage change
(sum(rate(http_requests_total[5m])) - sum(rate(http_requests_total[5m] offset 1h))) / sum(rate(http_requests_total[5m] offset 1h)) * 100

# Which series appeared in the last hour that did not exist an hour ago (new deployments, label explosions)
count by (job) (up) unless count by (job) (up offset 1h)

# Labels present on a metric without pulling values
group by (job, instance) (http_requests_total)

# Guard a division against zero: return nothing rather than NaN or Inf
sum(rate(a_total[5m])) / (sum(rate(b_total[5m])) > 0)

# Default when a series is missing: 0 if no matching series
sum(rate(http_requests_total{job="api"}[5m])) or vector(0)

# Per-pod CPU throttling ratio (above 25% sustained means the limit is too low)
sum by (pod) (rate(container_cpu_cfs_throttled_periods_total[5m])) / sum by (pod) (rate(container_cpu_cfs_periods_total[5m]))

# Node memory pressure without cache
1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes

# Disk read/write latency per operation from node_exporter
rate(node_disk_read_time_seconds_total[5m]) / rate(node_disk_reads_completed_total[5m])

# Time since a counter last moved (stalled consumer, frozen job)
time() - timestamp(max_over_time(jobs_processed_total[1h]) and changes(jobs_processed_total[1h]) > 0)

# Kubernetes: pods not ready for more than 15 minutes, excluding completed jobs
min_over_time(kube_pod_status_ready{condition="true"}[15m]) == 0 unless on (pod, namespace) kube_pod_status_phase{phase="Succeeded"} == 1

# Kubernetes: deployments with fewer available replicas than desired
kube_deployment_status_replicas_available < kube_deployment_spec_replicas

# Kubernetes: HPA at its ceiling
kube_horizontalpodautoscaler_status_current_replicas >= kube_horizontalpodautoscaler_spec_max_replicas

# Scrape health: targets whose scrape takes longer than half their interval
scrape_duration_seconds > 7.5   # for a 15 s interval

# Prometheus itself: samples ingested per second and head series
rate(prometheus_tsdb_head_samples_appended_total[5m])
prometheus_tsdb_head_series

# Percentiles across a fleet, keeping one label
histogram_quantile(0.95, sum by (le, region) (rate(http_request_duration_seconds_bucket[5m])))

# Subquery: the maximum 5 minute rate seen in the last day
max_over_time(sum(rate(http_requests_total[5m]))[1d:1m])

# Aggregation over labels whose names you do not know: quantile across instances
quantile by (job) (0.9, rate(process_cpu_seconds_total[5m]))

# Label manipulation: copy a label, then aggregate on the copy
sum by (env) (label_replace(rate(http_requests_total[5m]), "env", "$1", "namespace", "(prod|staging)-.*"))

# Info metric join: attach the image version to restart counts
increase(kube_pod_container_status_restarts_total[1h]) * on (pod, container, namespace) group_left (image) kube_pod_container_info

groups:
  - name: api
    interval: 30s
    rules:
      - alert: ApiHighErrorRate
        expr: |
          sum by (service) (rate(http_requests_total{status=~"5.."}[5m]))
            /
          sum by (service) (rate(http_requests_total[5m]))
            > 0.05
        for: 10m
        keep_firing_for: 5m
        labels:
          severity: page
        annotations:
          summary: "{{ $labels.service }} returning {{ $value | humanizePercentage }} errors"
          runbook_url: https://wiki.example.com/runbooks/api-errors

      - alert: TargetDown
        expr: up == 0
        for: 5m
        labels:
          severity: ticket

# tests.yaml, run with: promtool test rules tests.yaml
rule_files: [rules/api.yaml]
evaluation_interval: 30s
tests:
  - interval: 30s
    input_series:
      - series: 'http_requests_total{service="api", status="500"}'
        values: '0+10x40'              # starts at 0, +10 per interval, 40 samples
      - series: 'http_requests_total{service="api", status="200"}'
        values: '0+90x40'
    alert_rule_test:
      - eval_time: 12m                 # past the 10m for
        alertname: ApiHighErrorRate
        exp_alerts:
          - exp_labels: { service: api, severity: page }
            exp_annotations:
              summary: "api returning 10% errors"
              runbook_url: https://wiki.example.com/runbooks/api-errors
      - eval_time: 5m                  # still pending: expect nothing
        alertname: ApiHighErrorRate
        exp_alerts: []
    promql_expr_test:
      - expr: job:http_requests:rate5m
        eval_time: 5m
        exp_samples: [{ labels: 'job:http_requests:rate5m', value: 3.3333333333333335 }]   # sum by (job) drops service; no job label in the input

# alertmanager.yml
global:
  resolve_timeout: 5m
route:
  receiver: default-ticket
  group_by: [alertname, cluster, namespace]
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 12h
  routes:
    - matchers: [severity="page"]
      receiver: oncall-pager
      group_wait: 10s
      repeat_interval: 4h
      routes:
        - matchers: [team="db"]
          receiver: db-pager                # more specific route wins for db pages
    - matchers: [alertname=~"Watchdog|InfoInhibitor"]
      receiver: blackhole                   # alerts that exist to be present
    - matchers: [severity="ticket"]
      receiver: default-ticket
      active_time_intervals: [business-hours]
      continue: true                        # also fall through to the chat route
    - matchers: [severity=~"ticket|warning"]
      receiver: team-chat
time_intervals:
  - name: business-hours
    time_intervals:
      - weekdays: ["monday:friday"]
        times: [{ start_time: "09:00", end_time: "17:00" }]
        location: Australia/Sydney
inhibit_rules:
  - source_matchers: [alertname="NodeDown"]
    target_matchers: [severity=~"page|ticket"]
    equal: [instance]                       # silence everything else from a node that is down
  - source_matchers: [severity="page"]
    target_matchers: [severity="ticket"]
    equal: [alertname, namespace]           # do not ticket what is already paging
receivers:
  - name: blackhole
  - name: oncall-pager
    pagerduty_configs:
      - routing_key_file: /etc/alertmanager/secrets/pagerduty   # never inline the key
        severity: '{{ if eq .CommonLabels.severity "page" }}critical{{ else }}warning{{ end }}'
  - name: db-pager
    pagerduty_configs: [{ routing_key_file: /etc/alertmanager/secrets/pagerduty-db }]
  - name: default-ticket
    webhook_configs: [{ url_file: /etc/alertmanager/secrets/ticket-webhook, send_resolved: true }]
  - name: team-chat
    slack_configs:
      - api_url_file: /etc/alertmanager/secrets/slack
        channel: "#alerts"
        title: '{{ .GroupLabels.alertname }} ({{ .Alerts.Firing | len }} firing)'
        text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ "\n" }}{{ end }}'

amtool check-config alertmanager.yml                                        # parse and template check
amtool config routes test --config.file alertmanager.yml severity=page team=db   # which receiver a label set reaches
amtool config routes show --config.file alertmanager.yml                    # routing tree
amtool alert query --alertmanager.url http://localhost:9093 severity=page   # currently firing, as Alertmanager sees them
amtool alert query -o json | jq -r '.[] | [.labels.alertname, .status.state, (.status.inhibitedBy|length), (.status.silencedBy|length)] | @tsv'
amtool silence add --alertmanager.url http://localhost:9093 alertname=ApiHighErrorRate service=api -d 2h -c "deploy in progress" -a alice
amtool silence query; amtool silence expire <id>
amtool cluster show                                                          # HA peers and gossip state

scrape_configs:
  - job_name: kubernetes-pods
    kubernetes_sd_configs:
      - role: pod
        namespaces: { names: [my-namespace, monitoring] }
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: "true"                                          # opt-in by annotation
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)                                            # only when the annotation exists
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        target_label: __address__                              # swap the port for the annotated one
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)                # every pod label becomes a metric label
      - source_labels: [__meta_kubernetes_namespace]
        target_label: namespace
      - source_labels: [__meta_kubernetes_pod_name]
        target_label: pod
      - source_labels: [__meta_kubernetes_pod_container_init]
        action: drop
        regex: "true"
      - source_labels: [__meta_kubernetes_pod_phase]
        action: drop
        regex: Pending|Succeeded|Failed
      - action: hashmod                                        # shard targets across two servers
        source_labels: [__address__]
        modulus: 2
        target_label: __tmp_shard
      - source_labels: [__tmp_shard]
        action: keep
        regex: "0"
  - job_name: blackbox
    metrics_path: /probe
    params: { module: [http_2xx] }
    static_configs:
      - targets: [https://app.example.com/healthz, https://api.example.com/ready]
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target                           # the URL becomes the ?target= parameter
      - source_labels: [__param_target]
        target_label: instance                                 # keep it visible
      - target_label: __address__
        replacement: blackbox-exporter.monitoring:9115         # scrape the exporter, not the URL

remote_write:
  - url: https://mimir.example.com/api/v1/push
    name: mimir
    protobuf_message: io.prometheus.write.v2.Request     # remote write 2.0 (3.x); omit for 1.0 receivers
    headers: { X-Scope-OrgID: my-tenant }
    basic_auth:
      username: prometheus
      password_file: /etc/prometheus/secrets/remote-write   # never inline
    queue_config:
      capacity: 10000            # samples buffered per shard
      max_shards: 50
      max_samples_per_send: 2000
      batch_send_deadline: 5s
      min_backoff: 30ms
      max_backoff: 5s
      retry_on_http_429: true
    write_relabel_configs:       # runs before sending; drop what long-term storage does not need
      - source_labels: [__name__]
        regex: 'go_.*|process_.*'
        action: drop
    metadata_config: { send: true }

groups:
  - name: api-recording
    interval: 30s
    rules:
      - record: job:http_requests:rate5m
        expr: sum by (job) (rate(http_requests_total[5m]))

promtool check rules rules/*.yaml         # syntax and PromQL parse
promtool check config prometheus.yml      # also checks referenced rule files
promtool test rules tests.yaml            # unit tests with synthetic series

metric_relabel_configs:
  - source_labels: [__name__]
    regex: 'go_gc_duration_seconds.*'
    action: drop           # drop whole metrics
  - regex: 'request_id'
    action: labeldrop      # drop one label from every series; results must still be unique

scrape_configs:
  - job_name: api
    sample_limit: 50000     # scrape fails if exceeded
    label_limit: 30

# Targets that are not up, with the scrape error
curl -s localhost:9090/api/v1/targets | jq -r '.data.activeTargets[] | select(.health!="up") | [.labels.job, .scrapeUrl, .lastError] | @tsv'

# Metrics with the most series in the head block
curl -s localhost:9090/api/v1/status/tsdb | jq -r '.data.seriesCountByMetricName[] | [.value, .name] | @tsv'

# Labels with the most distinct values
curl -s localhost:9090/api/v1/status/tsdb | jq -r '.data.labelValueCountByLabelName[] | [.value, .name] | @tsv'

# Series count for one metric
curl -s 'localhost:9090/api/v1/series?match[]=http_requests_total' | jq '.data | length'

# Run an instant query
curl -s --data-urlencode 'query=sum by (job) (up)' localhost:9090/api/v1/query | jq -r '.data.result[] | [.metric.job, .value[1]] | @tsv'

# Evaluate at a past instant
curl -s --data-urlencode 'query=up' --data-urlencode "time=$(date -d '1 hour ago' +%s)" localhost:9090/api/v1/query | jq '.data.result | length'

# Rules that fail to evaluate
curl -s localhost:9090/api/v1/rules | jq -r '.data.groups[].rules[] | select(.health!="ok") | [.name, .lastError] | @tsv'

# Firing alerts, counted by name and severity
curl -s localhost:9090/api/v1/alerts | jq -r '.data.alerts[] | select(.state=="firing") | [.labels.alertname, .labels.severity] | @tsv' | sort | uniq -c

# Effective runtime flags
curl -s localhost:9090/api/v1/status/flags | jq

# Reload config and rules (needs --web.enable-lifecycle; otherwise send SIGHUP)
curl -sX POST localhost:9090/-/reload

# Build info and version
curl -s localhost:9090/api/v1/status/buildinfo | jq -r '.data | "\(.version) \(.goVersion)"'

# Runtime: head series, chunks, WAL replay status, retention in use
curl -s localhost:9090/api/v1/status/runtimeinfo | jq '.data | {timeSeriesCount, chunkCount, storageRetention, corruptionCount, reloadConfigSuccess}'

# Scrape pools and how many targets are healthy in each
curl -s localhost:9090/api/v1/targets | jq -r '.data.activeTargets | group_by(.scrapePool)[] | "\(.[0].scrapePool)\t\(map(select(.health=="up")) | length)/\(length)"'

# Targets discovered but dropped by relabelling, with the reason labels
curl -s 'localhost:9090/api/v1/targets?state=dropped' | jq -r '.data.droppedTargets[] | .discoveredLabels | "\(.__address__)\t\(.job // .__meta_kubernetes_namespace // "-")"' | head

# Metadata (type and help) for one metric across every target
curl -s 'localhost:9090/api/v1/metadata?metric=http_requests_total' | jq '.data'

# Every label name in use
curl -s localhost:9090/api/v1/labels | jq -r '.data[]'

# Values of one label, optionally restricted by a matcher
curl -s 'localhost:9090/api/v1/label/job/values' | jq -r '.data[]'
curl -s 'localhost:9090/api/v1/label/pod/values?match[]=up{namespace="my-namespace"}' | jq -r '.data[]'

# Series cardinality of a label combination without a PromQL query
curl -s 'localhost:9090/api/v1/series?match[]={__name__=~"http_.*",job="api"}' | jq '.data | group_by(.__name__) | map({name: .[0].__name__, series: length}) | sort_by(-.series)'

# Range query, printed as a CSV of time and value
curl -s --data-urlencode 'query=sum(rate(http_requests_total[5m]))' --data-urlencode "start=$(date -d '1 hour ago' +%s)" --data-urlencode "end=$(date +%s)" --data-urlencode 'step=60' localhost:9090/api/v1/query_range | jq -r '.data.result[0].values[] | "\(.[0] | todate),\(.[1])"'

# Instant query with a query timeout and evaluation statistics
curl -s --data-urlencode 'query=count({__name__=~".+"})' --data-urlencode 'timeout=30s' --data-urlencode 'stats=all' localhost:9090/api/v1/query | jq '.data.stats.timings'

# Alerting rules with their current state and active alert count
curl -s 'localhost:9090/api/v1/rules?type=alert' | jq -r '.data.groups[].rules[] | "\(.state)\t\(.alerts | length)\t\(.name)"' | sort

# Rule groups by evaluation time, slowest first
curl -s localhost:9090/api/v1/rules | jq -r '.data.groups[] | "\(.evaluationTime | . * 1000 | floor) ms\t\(.file):\(.name)"' | sort -rn | head

# Alerts pending longer than expected (activeAt older than 30 minutes)
curl -s localhost:9090/api/v1/alerts | jq -r --arg t "$(date -d '30 min ago' -u +%FT%TZ)" '.data.alerts[] | select(.state=="pending" and .activeAt < $t) | "\(.labels.alertname)\t\(.activeAt)"'

# Alertmanagers Prometheus is sending to, and any it dropped
curl -s localhost:9090/api/v1/alertmanagers | jq '.data | {active: [.activeAlertmanagers[].url], dropped: [.droppedAlertmanagers[].url]}'

# Configuration as loaded (secrets are redacted as <secret>)
curl -s localhost:9090/api/v1/status/config | jq -r .data.yaml | head -40

# WAL replay progress after a restart
curl -s localhost:9090/api/v1/status/walreplay | jq .data

# Health and readiness endpoints for probes
curl -sf localhost:9090/-/healthy && curl -sf localhost:9090/-/ready && echo ready

# TSDB snapshot for a backup (needs --web.enable-admin-api; writes under data/snapshots/)
curl -sX POST localhost:9090/api/v1/admin/tsdb/snapshot | jq -r .data.name

# Delete series matching a selector, then compact (admin API; irreversible)
curl -sX POST -g 'localhost:9090/api/v1/admin/tsdb/delete_series?match[]={job="old-job"}' && curl -sX POST localhost:9090/api/v1/admin/tsdb/clean_tombstones

# promtool: query the server from the shell
promtool query instant http://localhost:9090 'sum by (job) (up)'
promtool query range --start "$(date -d '1 hour ago' +%s)" --end "$(date +%s)" --step 5m http://localhost:9090 'sum(rate(http_requests_total[5m]))'

# promtool: label cardinality analysis of a running server
promtool tsdb analyze /var/lib/prometheus/data | head -60           # on the server; reads the blocks directly
promtool query labels http://localhost:9090 job                     # label values via the API

# promtool: scrape an endpoint once and check the exposition format
curl -s localhost:9100/metrics | promtool check metrics

# promtool: which metrics an exporter exposes, by type
curl -s localhost:9100/metrics | grep '^# TYPE' | awk '{print $4}' | sort | uniq -c

# promtool: test a relabel rule against a label set without a server
promtool check service-discovery prometheus.yml kubernetes-pods --timeout 30s | head -40

# Fan out one query to several servers
for s in prom-a prom-b; do printf '%s\t' "$s"; curl -s --data-urlencode 'query=count(up==1)' "http://$s.example.com:9090/api/v1/query" | jq -r '.data.result[0].value[1]'; done

# Metrics that have no HELP text (usually hand-rolled instrumentation)
curl -s localhost:9090/api/v1/metadata | jq -r '.data | to_entries[] | select(.value[0].help=="") | .key'

# Compare the label sets of two series to see why a join fails
curl -s --data-urlencode 'query=kube_pod_owner{pod="my-app-7c9d"}' localhost:9090/api/v1/query | jq '.data.result[].metric | keys'

#!/usr/bin/env bash
# cardinality-report.sh [prometheus-url] [top-n]: top metrics by series and the highest-cardinality label on each
set -euo pipefail
url=${1:-http://localhost:9090}; top=${2:-15}
total=$(curl -sf "$url/api/v1/status/tsdb" | jq '.data.headStats.numSeries')
printf 'head series: %s\n\n%-8s %-50s %s\n' "$total" SERIES METRIC "HIGHEST-CARDINALITY LABEL"
curl -sf "$url/api/v1/status/tsdb" | jq -r '.data.seriesCountByMetricName[] | "\(.value) \(.name)"' | head -n "$top" \
  | while read -r count name; do
    worst=$(curl -sf -g "$url/api/v1/series?match[]=$name" \
      | jq -r '[.data[] | to_entries[] | select(.key != "__name__")] | group_by(.key) | map({label: .[0].key, values: (map(.value) | unique | length)}) | max_by(.values) | "\(.label) (\(.values) values)"')
    printf '%-8s %-50s %s\n' "$count" "$name" "$worst"
  done

#!/usr/bin/env bash
# prom-health.sh [prometheus-url]: targets down, rules failing, config reload state, remote write lag; exit 1 on any finding
set -euo pipefail
url=${1:-http://localhost:9090}; rc=0
q() { curl -sf --max-time 10 --data-urlencode "query=$1" "$url/api/v1/query" | jq -r '.data.result[0].value[1] // "0"'; }

curl -sf --max-time 5 "$url/-/ready" >/dev/null || { echo "CRIT: not ready"; exit 2; }

if [ "$(curl -sf "$url/api/v1/status/runtimeinfo" | jq -r .data.reloadConfigSuccess)" != true ]; then
  echo "CRIT: last config reload failed"; rc=1
fi

down=$(curl -sf "$url/api/v1/targets" | jq -r '.data.activeTargets[] | select(.health!="up") | "  \(.labels.job)\t\(.scrapeUrl)\t\(.lastError)"')
[ -z "$down" ] || { echo "WARN: targets down:"; echo "$down"; rc=1; }

bad=$(curl -sf "$url/api/v1/rules" | jq -r '.data.groups[] | .name as $g | .rules[] | select(.health!="ok") | "  \($g)/\(.name)\t\(.lastError)"')
[ -z "$bad" ] || { echo "WARN: rules failing:"; echo "$bad"; rc=1; }

missed=$(q 'sum(increase(prometheus_rule_group_iterations_missed_total[1h]))')
[ "${missed%.*}" -eq 0 ] 2>/dev/null || { echo "WARN: $missed rule evaluations missed in the last hour (groups too slow)"; rc=1; }

lag=$(q 'max(prometheus_remote_storage_highest_timestamp_in_seconds - ignoring(remote_name, url) group_right prometheus_remote_storage_queue_highest_sent_timestamp_seconds)')
[ "${lag%.*}" -lt 60 ] 2>/dev/null || { echo "WARN: remote write lag ${lag}s"; rc=1; }

dropped=$(q 'sum(increase(prometheus_remote_storage_samples_dropped_total[1h]))')
[ "${dropped%.*}" -eq 0 ] 2>/dev/null || { echo "WARN: $dropped samples dropped by remote write in the last hour"; rc=1; }

[ $rc -eq 0 ] && echo "OK: $(q 'count(up==1)') targets up, $(q 'prometheus_tsdb_head_series') series"
exit $rc

#!/usr/bin/env bash
# silence.sh DURATION COMMENT matcher [matcher...]   e.g. silence.sh 2h "db-01 patching" alertname=~".*" instance="db-01:9100"
set -euo pipefail
: "${ALERTMANAGER_URL:=http://localhost:9093}"
duration=$1; comment=$2; shift 2
author=${USER:-unknown}
existing=$(amtool silence query --alertmanager.url "$ALERTMANAGER_URL" -o json "$@" | jq -r --arg a "$author" '.[] | select(.createdBy==$a) | .id')
for id in $existing; do amtool silence expire --alertmanager.url "$ALERTMANAGER_URL" "$id" && echo "expired earlier silence $id"; done
id=$(amtool silence add --alertmanager.url "$ALERTMANAGER_URL" -a "$author" -d "$duration" -c "$comment" "$@")
echo "silence $id active for $duration: $*"
amtool alert query --alertmanager.url "$ALERTMANAGER_URL" --silenced "$@" -o json | jq -r '.[] | "  now silenced: \(.labels.alertname) \(.labels.instance // "")"'