Elasticsearch

export ES_URL=https://localhost:9200
export ES_API_KEY='<encoded-api-key>'      # from Kibana or POST _security/api_key
es() { curl -sS --cacert http_ca.crt -H "Authorization: ApiKey $ES_API_KEY" -H 'Content-Type: application/json' "$@"; }

es "$ES_URL/"                               # cluster name and version
es "$ES_URL/_cluster/health?pretty"

es "$ES_URL/_cluster/health?pretty"
es "$ES_URL/_cat/indices?v&health=red&s=index"
es "$ES_URL/_cat/shards?v&s=state&h=index,shard,prirep,state,node,unassigned.reason" | grep -v STARTED
es "$ES_URL/_cluster/allocation/explain?pretty"         # no body: explains the first unassigned shard
es "$ES_URL/_cluster/allocation/explain?pretty" -d '{"index": "logs-2026.09.01", "shard": 0, "primary": true}'

es "$ES_URL/_cat/allocation?v&h=node,shards,disk.percent,disk.used,disk.avail"

# Temporary relief while you free space; remove afterwards by setting it to null
es -X PUT "$ES_URL/_cluster/settings" -d '{"persistent": {"cluster.routing.allocation.disk.watermark.flood_stage": "97%"}}'

PUT my-index
{
  "settings": { "number_of_shards": 1, "number_of_replicas": 1, "refresh_interval": "5s" },
  "mappings": {
    "dynamic": "strict",
    "properties": {
      "service":     { "type": "keyword" },
      "message":     { "type": "text" },
      "level":       { "type": "keyword" },
      "duration_ms": { "type": "float" },
      "@timestamp":  { "type": "date" },
      "host": {
        "properties": { "name": { "type": "keyword" }, "ip": { "type": "ip" } }
      }
    }
  }
}

es -X POST "$ES_URL/my-index/_doc" -d '{"service":"api","message":"timeout"}'           # auto ID
es -X PUT  "$ES_URL/my-index/_doc/1" -d '{"service":"api"}'                            # create or replace ID 1
es -X POST "$ES_URL/my-index/_update/1" -d '{"doc":{"level":"error"}}'                 # partial update
es -X POST "$ES_URL/_bulk" -H 'Content-Type: application/x-ndjson' --data-binary @bulk.ndjson

{"index":{"_index":"my-index","_id":"1"}}
{"service":"api","message":"timeout","@timestamp":"2026-09-01T00:00:00Z"}

es -X POST "$ES_URL/_bulk" -H 'Content-Type: application/x-ndjson' --data-binary @bulk.ndjson | jq '{errors, failed: [.items[] | .[] | select(.error) | .error.reason] | .[:5]}'

PUT my-index
{
  "mappings": {
    "properties": {
      "title":    { "type": "text", "fields": { "raw": { "type": "keyword", "ignore_above": 256 } } },
      "trace_id": { "type": "keyword", "doc_values": false },
      "payload":  { "type": "object", "enabled": false },
      "tags":     { "type": "keyword", "null_value": "none" },
      "price":    { "type": "scaled_float", "scaling_factor": 100 },
      "labels":   { "type": "flattened" },
      "items":    { "type": "nested", "properties": { "sku": { "type": "keyword" }, "qty": { "type": "integer" } } }
    },
    "runtime": {
      "duration_s": { "type": "double", "script": { "source": "emit(doc['duration_ms'].value / 1000.0)" } }
    }
  }
}

es -X PUT "$ES_URL/_component_template/logs-mappings" -d '{"template": {"mappings": {"dynamic": "strict", "properties": {"@timestamp": {"type": "date"}, "service": {"type": "keyword"}, "message": {"type": "text"}}}}}'
es -X PUT "$ES_URL/_index_template/logs-my-app" -d '{"index_patterns": ["logs-my-app-*"], "data_stream": {}, "priority": 200, "composed_of": ["logs-mappings"], "template": {"settings": {"index.lifecycle.name": "logs"}}}'
es -X POST "$ES_URL/_index_template/_simulate_index/logs-my-app-default?pretty"   # what a new index with this name would get, and which template won

PUT my-index
{
  "settings": {
    "analysis": {
      "char_filter": { "strip_html": { "type": "html_strip" } },
      "filter": {
        "english_stop": { "type": "stop", "stopwords": "_english_" },
        "edge_3_20":    { "type": "edge_ngram", "min_gram": 3, "max_gram": 20 }
      },
      "analyzer": {
        "message_index":  { "type": "custom", "char_filter": ["strip_html"], "tokenizer": "standard", "filter": ["lowercase", "asciifolding", "english_stop"] },
        "prefix_index":   { "type": "custom", "tokenizer": "standard", "filter": ["lowercase", "edge_3_20"] },
        "prefix_search":  { "type": "custom", "tokenizer": "standard", "filter": ["lowercase"] }
      },
      "normalizer": { "lc": { "type": "custom", "filter": ["lowercase", "asciifolding"] } }
    }
  },
  "mappings": {
    "properties": {
      "message":  { "type": "text", "analyzer": "message_index" },
      "hostname": { "type": "text", "analyzer": "prefix_index", "search_analyzer": "prefix_search" },
      "service":  { "type": "keyword", "normalizer": "lc" }
    }
  }
}

# Run text through a named analyser, a field's analyser, or an ad-hoc chain; explain shows each stage's output
es "$ES_URL/my-index/_analyze" -d '{"analyzer":"message_index","text":"<p>Connection timed out to Zürich</p>"}' | jq -r '.tokens[].token'
es "$ES_URL/my-index/_analyze" -d '{"field":"hostname","text":"web-01.example.com"}' | jq -r '.tokens[].token'
es "$ES_URL/_analyze" -d '{"tokenizer":"whitespace","filter":["lowercase",{"type":"stop","stopwords":["to"]}],"text":"Connection timed out to Zürich","explain":true}'

GET my-index/_search
{
  "size": 20,
  "query": {
    "bool": {
      "must":     [{ "match": { "message": "connection timeout" } }],
      "filter":   [
        { "term":  { "service": "api" } },
        { "range": { "@timestamp": { "gte": "now-1h" } } }
      ],
      "must_not": [{ "term": { "level": "debug" } }],
      "should":   [{ "match_phrase": { "message": "connection refused" } }]
    }
  },
  "sort": [{ "@timestamp": "desc" }],
  "_source": ["service", "message", "@timestamp"]
}

POST my-index/_pit?keep_alive=2m

GET _search
{
  "size": 1000,
  "pit": { "id": "<pit-id>", "keep_alive": "2m" },
  "sort": [{ "@timestamp": "asc" }],
  "search_after": [1788220800000]
}

GET my-index/_search
{
  "query": {
    "multi_match": {
      "query": "connection timeout",
      "fields": ["message^3", "title"],
      "type": "best_fields",
      "operator": "and"
    }
  },
  "highlight": { "fields": { "message": {} } }
}

GET my-index/_search
{
  "query": {
    "bool": {
      "filter": [
        { "terms":  { "service": ["api", "worker"] } },
        { "exists": { "field": "host.ip" } },
        { "prefix": { "host.name": "web-" } },
        { "range":  { "duration_ms": { "gte": 500, "lt": 2000 } } }
      ],
      "should": [
        { "match_phrase": { "message": { "query": "connection refused", "slop": 1 } } },
        { "match": { "message": { "query": "reset", "boost": 0.5 } } }
      ],
      "minimum_should_match": 1
    }
  }
}

GET my-index/_search
{
  "query": {
    "nested": {
      "path": "items",
      "query": { "bool": { "filter": [{ "term": { "items.sku": "ABC-1" } }, { "range": { "items.qty": { "gte": 10 } } }] } },
      "inner_hits": {}
    }
  }
}

GET my-index/_search
{
  "query": { "query_string": { "query": "service:api AND level:(error OR warn) AND NOT message:\"health check\"", "default_field": "message" } }
}

GET my-index/_search
{
  "size": 0,
  "aggs": {
    "by_service": {
      "terms": { "field": "service", "size": 10 },
      "aggs": {
        "p95":       { "percentiles": { "field": "duration_ms", "percents": [95] } },
        "over_time": { "date_histogram": { "field": "@timestamp", "fixed_interval": "5m" } }
      }
    }
  }
}

GET my-index/_search
{
  "size": 0,
  "query": { "range": { "@timestamp": { "gte": "now-24h" } } },
  "aggs": {
    "hosts":        { "cardinality": { "field": "host.name", "precision_threshold": 3000 } },
    "error_rate":   { "filters": { "filters": { "errors": { "term": { "level": "error" } }, "all": { "match_all": {} } } } },
    "latency":      { "range": { "field": "duration_ms", "ranges": [{ "to": 100 }, { "from": 100, "to": 1000 }, { "from": 1000 }] } },
    "slowest":      { "top_hits": { "size": 3, "sort": [{ "duration_ms": "desc" }], "_source": ["service", "message", "duration_ms"] } },
    "per_service":  {
      "terms": { "field": "service", "size": 50, "order": { "p99": "desc" } },
      "aggs": {
        "p99":  { "percentiles": { "field": "duration_ms", "percents": [99] } },
        "top3": { "bucket_sort": { "size": 3 } }
      }
    },
    "walk": {
      "composite": {
        "size": 1000,
        "sources": [{ "svc": { "terms": { "field": "service" } } }, { "host": { "terms": { "field": "host.name" } } }]
      }
    }
  }
}

es -X POST "$ES_URL/_reindex?wait_for_completion=false" -d '{"source": {"index": "my-index-v1"}, "dest": {"index": "my-index-v2"}}'
es "$ES_URL/_tasks/<task-id>?pretty"        # progress of the reindex task

es -X POST "$ES_URL/_aliases" -d '{
  "actions": [
    {"remove": {"index": "my-index-v1", "alias": "my-index"}},
    {"add":    {"index": "my-index-v2", "alias": "my-index", "is_write_index": true}}
  ]
}'

# Subset of documents, only some fields, into an index that already has the right mapping
es -X POST "$ES_URL/_reindex?wait_for_completion=false&slices=auto" -d '{
  "conflicts": "proceed",
  "source": { "index": "logs-2026.09.*", "query": { "range": { "@timestamp": { "gte": "2026-09-01" } } }, "_source": ["@timestamp", "service", "message"] },
  "dest":   { "index": "logs-archive", "op_type": "create" }
}'

# Transform on the way through: rename a field and drop another
es -X POST "$ES_URL/_reindex" -d '{
  "source": { "index": "my-index-v1" },
  "dest":   { "index": "my-index-v2" },
  "script": { "lang": "painless", "source": "ctx._source.service = ctx._source.remove(\"svc\"); ctx._source.remove(\"debug\")" }
}'

# From another cluster; the remote host must be listed in reindex.remote.whitelist on this cluster
es -X POST "$ES_URL/_reindex" -d '{"source": {"remote": {"host": "https://old.example.com:9200", "username": "reindex"}, "index": "my-index"}, "dest": {"index": "my-index"}}'

es -X POST "$ES_URL/_reindex/<task-id>/_rethrottle?requests_per_second=500"   # slow a running reindex down; -1 removes the limit
es "$ES_URL/_tasks?actions=*reindex&detailed" | jq '.nodes[].tasks[] | {description, status: .status | {total, created, updated, version_conflicts}}'

es -X PUT "$ES_URL/_ilm/policy/logs" -d '{
  "policy": {
    "phases": {
      "hot":    { "actions": { "rollover": { "max_primary_shard_size": "50gb", "max_age": "1d" }, "set_priority": { "priority": 100 } } },
      "warm":   { "min_age": "2d", "actions": { "shrink": { "number_of_shards": 1 }, "forcemerge": { "max_num_segments": 1 }, "allocate": { "number_of_replicas": 1, "require": { "data": "warm" } }, "set_priority": { "priority": 50 } } },
      "cold":   { "min_age": "14d", "actions": { "allocate": { "number_of_replicas": 0, "require": { "data": "cold" } }, "set_priority": { "priority": 0 } } },
      "delete": { "min_age": "30d", "actions": { "delete": {} } }
    }
  }
}'

es -X PUT "$ES_URL/my-index-000001" -d '{"aliases": {"my-index": {"is_write_index": true}}}'
es -X POST "$ES_URL/my-index/_rollover?dry_run=true" -d '{"conditions": {"max_age": "1d", "max_primary_shard_size": "50gb"}}'   # would it roll now, and why
es -X PUT "$ES_URL/_cluster/settings" -d '{"persistent": {"indices.lifecycle.poll_interval": "1m"}}'                        # default 10m; ILM only checks conditions this often
es "$ES_URL/my-index-*/_ilm/explain?pretty&only_errors=true"
es -X POST "$ES_URL/my-index-000003/_ilm/retry"                                                                              # after fixing the cause of an ERROR step
es -X POST "$ES_URL/_ilm/move/my-index-000002" -d '{"current_step": {"phase": "hot", "action": "complete", "name": "complete"}, "next_step": {"phase": "warm"}}'   # force a phase change

es "$ES_URL/_ilm/policy/logs?pretty"
es "$ES_URL/my-index/_ilm/explain?pretty"        # current phase, step and any ERROR step
es "$ES_URL/_data_stream/logs-my-app-default?pretty"

es "$ES_URL/_cat/health?v&ts=false"
es "$ES_URL/_cat/nodes?v&h=name,node.role,master,heap.percent,ram.percent,cpu,load_1m,disk.used_percent,version&s=name"
es "$ES_URL/_cat/indices?v&h=index,health,status,pri,rep,docs.count,store.size,pri.store.size&s=pri.store.size:desc&bytes=gb"
es "$ES_URL/_cat/shards/logs-*?v&h=index,shard,prirep,state,docs,store,node&s=store:desc&bytes=mb"
es "$ES_URL/_cat/segments/my-index?v&h=index,shard,segment,docs.count,size&s=size:desc"   # many tiny segments means forcemerge or a busier merge scheduler
es "$ES_URL/_cat/recovery?v&active_only=true&h=index,shard,type,stage,source_node,target_node,bytes_percent,time"
es "$ES_URL/_cat/thread_pool?v&h=node_name,name,active,queue,rejected,completed&s=rejected:desc"
es "$ES_URL/_cat/fielddata?v&s=size:desc"        # heap held by fielddata; anything large here is a text field being aggregated
es "$ES_URL/_cat/templates?v&s=order"
es "$ES_URL/_cat/aliases?v&s=alias"
es "$ES_URL/_cat/tasks?v&detailed&s=running_time:desc"
es "$ES_URL/_cat/master?v"
es "$ES_URL/_cat/count/my-index?v"
es "$ES_URL/_cat/indices?h=index&s=index" | wc -l   # column-only output is script-friendly without jq

es "$ES_URL/_nodes/stats/jvm,indices?pretty" | jq '.nodes[] | {name, heap: .jvm.mem.heap_used_percent, old_gc: .jvm.gc.collectors.old.collection_count, query_ms: .indices.search.query_time_in_millis}'
es "$ES_URL/_cat/thread_pool/search,write?v&h=node_name,name,active,queue,rejected"
es "$ES_URL/_nodes/hot_threads"

# Search and indexing slow logs for one index
es -X PUT "$ES_URL/my-index/_settings" -d '{
  "index.search.slowlog.threshold.query.warn": "2s",
  "index.indexing.slowlog.threshold.index.warn": "1s"
}'

# Register an S3 repository (credentials go in the Elasticsearch keystore, not the request)
es -X PUT "$ES_URL/_snapshot/backups" -d '{"type": "s3", "settings": {"bucket": "my-es-backups"}}'
es -X POST "$ES_URL/_snapshot/backups/_verify?pretty"

es -X PUT "$ES_URL/_snapshot/backups/snap-$(date +%F)?wait_for_completion=false"
es "$ES_URL/_cat/snapshots/backups?v"

# Restore one index under a new name so it does not collide with the live index
es -X POST "$ES_URL/_snapshot/backups/snap-2026-09-01/_restore" -d '{"indices": "my-index", "rename_pattern": "(.+)", "rename_replacement": "restored-$1"}'

# Cluster status on one line
es "$ES_URL/_cluster/health" | jq -r '[.status, .number_of_nodes, .active_shards, .unassigned_shards] | @tsv'

# Indices sorted by size in bytes
es "$ES_URL/_cat/indices?h=index,store.size,docs.count&bytes=b" | sort -k2 -rn | head

# Shards that are not STARTED, with reasons
es "$ES_URL/_cat/shards?h=index,shard,prirep,state,unassigned.reason" | awk '$4!="STARTED"'

# Retry allocations that hit the retry limit (after fixing the cause)
es -X POST "$ES_URL/_cluster/reroute?retry_failed=true" -o /dev/null -w '%{http_code}\n'

# Mapping of one field across all indices
es "$ES_URL/_mapping/field/service?pretty"

# Tokens the analyser produces for a string
es "$ES_URL/my-index/_analyze" -d '{"field":"message","text":"Connection timed out"}' | jq -r '.tokens[].token'

# Document count per value of a keyword field
es "$ES_URL/my-index/_search" -d '{"size":0,"aggs":{"s":{"terms":{"field":"service","size":20}}}}' | jq -r '.aggregations.s.buckets[] | [.key, .doc_count] | @tsv'

# Cancel searches running longer than 30 seconds
es "$ES_URL/_tasks?actions=*search*&detailed" | jq -r '.nodes[].tasks | to_entries[] | select(.value.running_time_in_nanos > 30e9) | .key' | xargs -r -I{} curl -sS --cacert http_ca.crt -H "Authorization: ApiKey $ES_API_KEY" -X POST "$ES_URL/_tasks/{}/_cancel"

# Count documents from the last hour
es "$ES_URL/my-index/_count" -d '{"query":{"range":{"@timestamp":{"gte":"now-1h"}}}}' | jq .count

# Which node is the elected master, and every master-eligible node
es "$ES_URL/_cat/master?h=node"; es "$ES_URL/_cat/nodes?h=name,node.role" | awk '$2 ~ /m/'

# Nodes above 75% heap
es "$ES_URL/_cat/nodes?h=name,heap.percent" | awk '$2 > 75'

# Indices with more than one primary shard per node's worth of data (candidates for shrink)
es "$ES_URL/_cat/indices?h=index,pri,pri.store.size&bytes=gb" | awk '$2 > 1 && $3/$2 < 10 {print $1, $2, $3"gb"}'

# Total shard count against the default per-node limit of 1000
es "$ES_URL/_cluster/health" | jq '{active_shards, nodes: .number_of_data_nodes, per_node: (.active_shards / .number_of_data_nodes | floor)}'

# Every non-default persistent and transient cluster setting
es "$ES_URL/_cluster/settings?flat_settings" | jq '.persistent + .transient'

# Every index setting that differs from the defaults, one index
es "$ES_URL/my-index/_settings?flat_settings" | jq '.[].settings'

# Which index template a new index name would use
es -X POST "$ES_URL/_index_template/_simulate_index/logs-my-app-2026.10.01" | jq '{overlapping, template: .template.settings}'

# ILM phase of every index, grouped
es "$ES_URL/logs-*/_ilm/explain" | jq -r '.indices[] | [.index, .phase, .action, .step] | @tsv' | sort -k2

# Indices in an ILM ERROR step with the failure reason
es "$ES_URL/*/_ilm/explain?only_errors=true" | jq -r '.indices[] | "\(.index)\t\(.step_info.reason // .step_info.type)"'

# Data streams and their backing index count
es "$ES_URL/_data_stream" | jq -r '.data_streams[] | [.name, (.indices | length), .ilm_policy // .lifecycle.data_retention] | @tsv'

# Force a data stream to roll over now (for example after changing the template mapping)
es -X POST "$ES_URL/logs-my-app-default/_rollover" | jq '{old_index, new_index, rolled_over}'

# Segments per shard for an index; hundreds means merges are behind
es "$ES_URL/_cat/segments/my-index?h=shard,segment" | awk '{c[$1]++} END {for (s in c) print s, c[s]}'

# Fields in an index, flattened, to spot field explosion (limit is index.mapping.total_fields.limit, default 1000)
es "$ES_URL/my-index/_mapping" | jq '[.. | objects | select(has("type")) ] | length'

# Recovery progress for shards currently moving or rebuilding
es "$ES_URL/_cat/recovery?active_only=true&h=index,shard,stage,source_node,target_node,bytes_percent,translog_ops_percent"

# Move a shard off a node explicitly (allocation may move it straight back if deciders disagree)
es -X POST "$ES_URL/_cluster/reroute" -d '{"commands":[{"move":{"index":"my-index","shard":0,"from_node":"es-data-1","to_node":"es-data-2"}}]}' | jq .acknowledged

# Drain a node before maintenance: exclude it, then watch its shard count fall to 0
es -X PUT "$ES_URL/_cluster/settings" -d '{"transient":{"cluster.routing.allocation.exclude._name":"es-data-3"}}'; watch -n5 "curl -sS --cacert http_ca.crt -H \"Authorization: ApiKey $ES_API_KEY\" '$ES_URL/_cat/allocation?h=node,shards' | grep es-data-3"

# Undo the drain
es -X PUT "$ES_URL/_cluster/settings" -d '{"transient":{"cluster.routing.allocation.exclude._name":null}}'

# Disable replica allocation for a rolling restart (primaries still allocate), then re-enable
es -X PUT "$ES_URL/_cluster/settings" -d '{"persistent":{"cluster.routing.allocation.enable":"primaries"}}'
es -X PUT "$ES_URL/_cluster/settings" -d '{"persistent":{"cluster.routing.allocation.enable":null}}'

# Flush before stopping a node so recovery has less translog to replay
es -X POST "$ES_URL/_flush"

# Slow searches from the slow log, grouped by index
journalctl -u elasticsearch --since -1h | grep 'index.search.slowlog' | grep -o '\[[^]]*\]\[[0-9]*\]' | sort | uniq -c | sort -rn | head

# Long-running tasks of any kind on any node
es "$ES_URL/_tasks?detailed" | jq -r '.nodes[].tasks[] | select(.running_time_in_nanos > 60e9) | [(.running_time_in_nanos/1e9|floor), .action, .description[:80]] | @tsv' | sort -rn

# Estimate how many hits a query has without fetching any
es "$ES_URL/my-index/_search?size=0&track_total_hits=true" -d '{"query":{"match":{"message":"timeout"}}}' | jq .hits.total.value

# Profile a slow query: per-shard time by query component
es "$ES_URL/my-index/_search" -d '{"profile":true,"size":0,"query":{"match":{"message":"timeout"}}}' | jq '[.profile.shards[].searches[].query[] | {type, time_in_nanos, description: .description[:60]}] | sort_by(-.time_in_nanos) | .[:5]'

# Create an API key limited to reading one index pattern, valid for 30 days
es -X POST "$ES_URL/_security/api_key" -d '{"name":"readonly-logs","expiration":"30d","role_descriptors":{"ro":{"indices":[{"names":["logs-*"],"privileges":["read","view_index_metadata"]}]}}}' | jq -r .encoded

# Who am I and what roles do I have
es "$ES_URL/_security/_authenticate" | jq '{username, roles}'

es -X POST "$ES_URL/my-index/_delete_by_query?conflicts=proceed" -d '{"query":{"term":{"level":"debug"}}}' | jq '{deleted, failures}'

#!/usr/bin/env bash
# usage: ES_URL=https://es.example.com:9200 ES_API_KEY=... es-report.sh [ca.crt]
set -euo pipefail
: "${ES_URL:?}" "${ES_API_KEY:?}"
ca=${1:-http_ca.crt}
es() { curl -sS --max-time 20 --cacert "$ca" -H "Authorization: ApiKey $ES_API_KEY" -H 'Content-Type: application/json' "$@"; }

es "$ES_URL/_cluster/health" | jq -r '"cluster \(.cluster_name): \(.status)  nodes=\(.number_of_nodes) shards=\(.active_shards) unassigned=\(.unassigned_shards) pending_tasks=\(.number_of_pending_tasks)"'

printf '\n-- nodes over 75%% heap or 85%% disk --\n'
es "$ES_URL/_cat/nodes?h=name,heap.percent,disk.used_percent,cpu,load_1m" | awk '$2 > 75 || $3 > 85' || true

printf '\n-- unassigned shards --\n'
es "$ES_URL/_cat/shards?h=index,shard,prirep,state,unassigned.reason" | awk '$4 == "UNASSIGNED"' | head -20 || true
if es "$ES_URL/_cat/shards?h=state" | grep -q UNASSIGNED; then
  es "$ES_URL/_cluster/allocation/explain" | jq -r '"first unassigned: \(.index)[\(.shard)] \(.allocate_explanation // .unassigned_info.reason)"'
fi

printf '\n-- indices with primary shards over 50gb or under 1gb (and more than 1 primary) --\n'
es "$ES_URL/_cat/indices?h=index,pri,pri.store.size&bytes=gb&s=pri.store.size:desc" \
  | awk '$3 > 0 { per = $3 / $2; if (per > 50 || ($2 > 1 && per < 1)) printf "%-50s pri=%s avg=%.1fgb\n", $1, $2, per }' || true

printf '\n-- thread pool rejections --\n'
es "$ES_URL/_cat/thread_pool/search,write,get?h=node_name,name,rejected" | awk '$3 > 0' || true

printf '\n-- ILM errors --\n'
es "$ES_URL/*/_ilm/explain?only_errors=true" | jq -r '.indices[] | "\(.index): \(.step_info.reason // .step_info.type // "unknown")"' || true

#!/usr/bin/env bash
# usage: es-prune.sh [--apply] PATTERN DAYS   e.g. es-prune.sh 'logs-legacy-*' 90
set -euo pipefail
apply=0; [[ ${1:-} == --apply ]] && { apply=1; shift; }
pattern=${1:?index pattern required}; days=${2:?days required}
[[ $days =~ ^[0-9]+$ ]] || { echo 'DAYS must be an integer' >&2; exit 2; }
es() { curl -sS --max-time 30 --cacert "${ES_CA:-http_ca.crt}" -H "Authorization: ApiKey $ES_API_KEY" -H 'Content-Type: application/json' "$@"; }
cutoff=$(date -u -d "-${days} days" +%s)

mapfile -t indices < <(es "$ES_URL/_cat/indices/$pattern?h=index&s=index")
(( ${#indices[@]} )) || { echo "no indices match $pattern" >&2; exit 0; }

to_delete=()
for idx in "${indices[@]}"; do
  # Skip anything that is a write target or under ILM; those are someone else's job
  if es "$ES_URL/$idx/_ilm/explain" | jq -e '.indices[].managed' >/dev/null 2>&1; then printf 'skip %s: ILM managed\n' "$idx"; continue; fi
  if es "$ES_URL/_alias/$idx" 2>/dev/null | jq -e '.[].aliases[] | select(.is_write_index == true)' >/dev/null; then printf 'skip %s: write index\n' "$idx"; continue; fi
  newest=$(es "$ES_URL/$idx/_search?size=0" -d '{"aggs":{"m":{"max":{"field":"@timestamp"}}}}' | jq -r '.aggregations.m.value // empty')
  [[ -n $newest ]] || { printf 'skip %s: no @timestamp\n' "$idx"; continue; }
  newest_s=$(( ${newest%.*} / 1000 ))
  if (( newest_s < cutoff )); then
    printf 'delete %s (newest doc %s)\n' "$idx" "$(date -u -d "@$newest_s" +%F)"
    to_delete+=("$idx")
  fi
done

(( ${#to_delete[@]} )) || { echo 'nothing to delete'; exit 0; }
(( apply )) || { printf '\n%d indices would be deleted; rerun with --apply\n' "${#to_delete[@]}"; exit 0; }
for idx in "${to_delete[@]}"; do
  es -X DELETE "$ES_URL/$idx" | jq -c --arg i "$idx" '{index: $i, acknowledged}'
done

#!/usr/bin/env python3
"""usage: ES_URL=... ES_API_KEY=... es-export.py INDEX 'QUERY_JSON' > out.ndjson"""
import json, os, sys, urllib.request, ssl

url, key = os.environ["ES_URL"], os.environ["ES_API_KEY"]
index, query = sys.argv[1], json.loads(sys.argv[2] if len(sys.argv) > 2 else '{"match_all": {}}')
ctx = ssl.create_default_context(cafile=os.environ.get("ES_CA", "http_ca.crt"))

def call(method, path, body=None):
    req = urllib.request.Request(url + path, method=method, data=json.dumps(body).encode() if body else None,
                                 headers={"Authorization": f"ApiKey {key}", "Content-Type": "application/json"})
    with urllib.request.urlopen(req, context=ctx, timeout=60) as r:
        return json.load(r)

pit = call("POST", f"/{index}/_pit?keep_alive=2m")["id"]
search_after, total = None, 0
try:
    while True:
        body = {"size": 1000, "query": query, "pit": {"id": pit, "keep_alive": "2m"},
                "sort": [{"@timestamp": "asc"}, {"_shard_doc": "asc"}]}   # _shard_doc breaks ties so no hit is skipped
        if search_after:
            body["search_after"] = search_after
        res = call("GET", "/_search", body)
        hits = res["hits"]["hits"]
        if not hits:
            break
        for h in hits:
            sys.stdout.write(json.dumps(h["_source"]) + "\n")
        total += len(hits)
        search_after = hits[-1]["sort"]
        pit = res["pit_id"]                                                # the PIT id can change between pages
finally:
    call("DELETE", "/_pit", {"id": pit})
print(f"exported {total} documents", file=sys.stderr)