Traefik

curl -s localhost:8080/api/overview | jq                     # counts, warnings, errors per type and enabled providers
curl -s localhost:8080/api/http/routers | jq -r '.[] | [.name, .rule, .status] | @tsv'
curl -s localhost:8080/api/http/services | jq -r '.[] | [.name, .status, ((.serverStatus // {}) | tostring)] | @tsv'
curl -s localhost:8080/api/http/middlewares | jq -r '.[] | [.name, .status] | @tsv'
curl -s localhost:8080/api/rawdata | jq                      # the full dynamic config Traefik is running

# traefik.yaml, read once at start-up
entryPoints:
  web:
    address: ":80"
    http:
      redirections:
        entryPoint: { to: websecure, scheme: https, permanent: true }
  websecure:
    address: ":443"
    http:
      tls:
        certResolver: letsencrypt
    transport:
      respondingTimeouts: { readTimeout: 60s, writeTimeout: 0s, idleTimeout: 180s }   # v3 defaults
    forwardedHeaders:
      trustedIPs: ["198.51.100.0/24"]   # only when a load balancer in front sets X-Forwarded-For

providers:
  docker:
    exposedByDefault: false          # containers need traefik.enable=true
  kubernetesCRD:
    allowCrossNamespace: false
  file:
    directory: /etc/traefik/dynamic
    watch: true

certificatesResolvers:
  letsencrypt:
    acme:
      email: ops@example.com
      storage: /data/acme.json       # must persist across restarts; keep mode 600
      tlsChallenge: {}

api:
  dashboard: true
ping: {}
accessLog:
  format: json
  filters: { statusCodes: ["400-599"], retryAttempts: true, minDuration: 1s }
log:
  level: INFO

http:
  routers:
    api:
      rule: "Host(`api.example.com`) && PathPrefix(`/v1`)"
      entryPoints: [websecure]
      middlewares: [ratelimit, secure-headers]
      service: api
      priority: 100
      tls: { certResolver: letsencrypt }

http:
  services:
    api:
      loadBalancer:
        servers:
          - url: "http://192.0.2.5:8080"
          - url: "http://192.0.2.6:8080"
        healthCheck: { path: /healthz, interval: 10s, timeout: 3s }
        sticky:
          cookie: { name: srv, httpOnly: true, secure: true, sameSite: lax }
    api-split:
      weighted:
        services:
          - { name: api-v1, weight: 90 }
          - { name: api-v2, weight: 10 }
    api-mirror:
      mirroring:
        service: api-v1
        mirrors: [{ name: api-v2, percent: 10 }]

http:
  middlewares:
    secure-headers:
      headers:
        stsSeconds: 31536000
        frameDeny: true
        contentTypeNosniff: true
        referrerPolicy: no-referrer
    ratelimit:
      rateLimit: { average: 100, burst: 200, period: 1s }
    basic-auth:
      basicAuth: { usersFile: /etc/traefik/users }        # htpasswd format
    forward-auth:
      forwardAuth:
        address: http://auth:4180/verify
        authResponseHeaders: [X-Auth-User, X-Auth-Groups]
    strip:
      stripPrefix: { prefixes: ["/api"] }
    retry:
      retry: { attempts: 3, initialInterval: 100ms }
    circuit:
      circuitBreaker: { expression: "NetworkErrorRatio() > 0.30 || ResponseCodeRatio(500, 600, 0, 600) > 0.25" }
    compress:
      compress: {}
    allowlist:
      ipAllowList:
        sourceRange: ["192.0.2.0/24", "198.51.100.0/24"]
        ipStrategy: { depth: 1 }     # only when behind one trusted proxy; see below

http:
  middlewares:
    public:
      chain:
        middlewares: [secure-headers, ratelimit, compress]
    upload-limit:
      buffering: { maxRequestBodyBytes: 104857600 }   # 100 MiB, 413 above
    cors:
      headers:
        accessControlAllowOriginList: ["https://app.example.com"]
        accessControlAllowMethods: [GET, POST, OPTIONS]
        accessControlAllowHeaders: [Authorization, Content-Type]
        accessControlMaxAge: 600
        addVaryHeader: true
    inflight:
      inFlightReq: { amount: 50 }
    error-pages:
      errors:
        status: ["500-599"]
        service: error-pages
        query: "/{status}.html"

tls:
  options:
    default:
      minVersion: VersionTLS12
      sniStrict: true                          # refuse connections whose SNI matches no certificate instead of serving the default
      cipherSuites:                            # TLS 1.2 only; TLS 1.3 suites are not configurable in Go
        - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
        - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
        - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
        - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
        - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
        - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
      curvePreferences: [X25519, CurveP256]
      alpnProtocols: [h2, http/1.1]
    mtls:
      minVersion: VersionTLS13
      clientAuth:
        caFiles: [/etc/traefik/certs/client-ca.pem]
        clientAuthType: RequireAndVerifyClientCert   # NoClientCert, RequestClientCert, RequireAnyClientCert, VerifyClientCertIfGiven
  certificates:                                # static certificates, matched to SNI automatically
    - certFile: /etc/traefik/certs/example.com.crt
      keyFile: /etc/traefik/certs/example.com.key
  stores:
    default:
      defaultCertificate:                      # served when no certificate matches and sniStrict is off
        certFile: /etc/traefik/certs/default.crt
        keyFile: /etc/traefik/certs/default.key
http:
  routers:
    admin:
      rule: "Host(`admin.example.com`)"
      service: admin
      tls: { options: mtls }                   # mtls@file from another provider
      middlewares: [client-cert-info]
  middlewares:
    client-cert-info:
      passTLSClientCert: { info: { subject: { commonName: true, organization: true } } }

http:
  serversTransports:
    internal-ca:
      serverName: my-app.internal              # SNI and name verified on the backend certificate
      rootCAs: [/etc/traefik/certs/internal-ca.pem]
      certificates: [{ certFile: /etc/traefik/certs/traefik-client.crt, keyFile: /etc/traefik/certs/traefik-client.key }]
      insecureSkipVerify: false
      maxIdleConnsPerHost: 200
      forwardingTimeouts: { dialTimeout: 5s, responseHeaderTimeout: 30s, idleConnTimeout: 90s }
  services:
    my-app:
      loadBalancer:
        serversTransport: internal-ca
        servers: [{ url: "https://192.0.2.5:8443" }]

# /etc/traefik/dynamic/apps.yaml
http:
  routers:
    app:
      rule: "Host(`app.example.com`)"
      entryPoints: [websecure]
      middlewares: [public@file]
      service: app
      tls: { certResolver: letsencrypt, domains: [{ main: example.com, sans: ["*.example.com"] }] }   # wildcard via DNS-01
    app-canary:
      rule: "Host(`app.example.com`) && Header(`X-Canary`, `always`)"
      entryPoints: [websecure]
      priority: 200
      service: app-v2
      tls: {}
    legacy:
      rule: "Host(`old.example.com`)"
      entryPoints: [websecure]
      middlewares: [to-new]
      service: noop@internal                   # a redirect needs no backend
      tls: {}
  middlewares:
    to-new:
      redirectRegex:
        regex: "^https://old\\.example\\.com/(.*)"
        replacement: "https://app.example.com/${1}"
        permanent: true
  services:
    app:
      weighted:
        services: [{ name: app-v1, weight: 95 }, { name: app-v2, weight: 5 }]
    app-v1:
      loadBalancer:
        servers: [{ url: "http://192.0.2.5:8080" }, { url: "http://192.0.2.6:8080" }]
        healthCheck: { path: /healthz, interval: 10s, timeout: 3s, followRedirects: false }
        passHostHeader: true
    app-v2:
      loadBalancer:
        servers: [{ url: "http://192.0.2.7:8080" }]
tcp:
  routers:
    db:
      rule: "HostSNI(`db.example.com`)"
      entryPoints: [postgres]                   # a TCP entrypoint declared in static config, address ":5432"
      service: db
      tls: { passthrough: true }                # SNI routing without terminating
  services:
    db:
      loadBalancer:
        servers: [{ address: "192.0.2.20:5432" }]

apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata: { name: api, namespace: my-namespace }
spec:
  entryPoints: [websecure]
  routes:
    - match: Host(`api.example.com`) && PathPrefix(`/v1`)
      kind: Rule
      priority: 100
      middlewares:
        - { name: secure-headers }
      services:
        - { name: api, port: 80 }
  tls: { certResolver: letsencrypt }

apiVersion: traefik.io/v1alpha1
kind: Middleware
metadata: { name: secure-headers, namespace: my-namespace }
spec:
  headers: { stsSeconds: 31536000, frameDeny: true }

apiVersion: traefik.io/v1alpha1
kind: TraefikService
metadata: { name: app-split, namespace: my-namespace }
spec:
  weighted:
    services:
      - { name: app-v1, port: 80, weight: 95 }
      - { name: app-v2, port: 80, weight: 5 }
    sticky: { cookie: { name: app, secure: true, httpOnly: true } }
---
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata: { name: app, namespace: my-namespace }
spec:
  entryPoints: [websecure]
  routes:
    - match: Host(`app.example.com`) && Header(`X-Canary`, `always`)
      kind: Rule
      priority: 200
      services: [{ name: app-v2, port: 80 }]
    - match: Host(`app.example.com`)
      kind: Rule
      middlewares: [{ name: public, namespace: traefik }]     # cross-namespace needs allowCrossNamespace
      services:
        - { name: app-split, kind: TraefikService }             # kind defaults to Service
  tls:
    secretName: app-example-com-tls                             # cert-manager Secret; omit for certResolver
    options: { name: strict, namespace: traefik }
---
apiVersion: traefik.io/v1alpha1
kind: TLSOption
metadata: { name: strict, namespace: traefik }
spec:
  minVersion: VersionTLS13
  sniStrict: true
---
apiVersion: traefik.io/v1alpha1
kind: ServersTransport
metadata: { name: internal-ca, namespace: my-namespace }
spec:
  serverName: my-app.my-namespace.svc
  rootCAsSecrets: [internal-ca]                                 # Secret with ca.crt
---
apiVersion: traefik.io/v1alpha1
kind: IngressRouteTCP
metadata: { name: db, namespace: my-namespace }
spec:
  entryPoints: [postgres]
  routes:
    - match: HostSNI(`db.example.com`)
      services: [{ name: my-db, port: 5432 }]
  tls: { passthrough: true }

kubectl get ingressroute,ingressroutetcp,middleware,traefikservice,tlsoption -A
kubectl get crd | grep traefik.io                                  # v3 group; traefik.containo.us means old CRDs
kubectl logs -n traefik deploy/traefik | grep -iE 'error|warn' | tail -20
kubectl port-forward -n traefik deploy/traefik 8080:8080           # API on localhost:8080 when api.insecure is off but the port exists

services:
  app:
    image: my-app:1.0
    labels:
      - traefik.enable=true
      - traefik.http.routers.app.rule=Host(`app.example.com`)
      - traefik.http.routers.app.entrypoints=websecure
      - traefik.http.routers.app.tls.certresolver=letsencrypt
      - traefik.http.routers.app.middlewares=app-strip
      - traefik.http.middlewares.app-strip.stripprefix.prefixes=/api
      - traefik.http.services.app.loadbalancer.server.port=8080
      - traefik.docker.network=edge
    networks: [edge]

http:
  routers:
    dashboard:
      rule: "Host(`traefik.example.com`) && (PathPrefix(`/api`) || PathPrefix(`/dashboard`))"
      entryPoints: [websecure]
      service: api@internal
      middlewares: [basic-auth]
      tls: { certResolver: letsencrypt }

metrics:
  prometheus:
    buckets: [0.1, 0.3, 1.2, 5.0]      # the default
    addEntryPointsLabels: true         # default true
    addRoutersLabels: true             # default false
    addServicesLabels: true            # default true
tracing:
  otlp:
    http: { endpoint: http://otel-collector:4318/v1/traces }

# 5xx ratio per service
sum by (service) (rate(traefik_service_requests_total{code=~"5.."}[5m]))
  / sum by (service) (rate(traefik_service_requests_total[5m]))

# Days until each certificate expires
(traefik_tls_certs_not_after - time()) / 86400

# Backends removed by health checks right now
traefik_service_server_up == 0

# p95 latency per service
histogram_quantile(0.95, sum by (le, service) (rate(traefik_service_request_duration_seconds_bucket[5m])))

# Certificates expiring within 14 days
(traefik_tls_certs_not_after - time()) / 86400 < 14

# Requests that matched no router (404 at the entrypoint, not from a service)
sum(rate(traefik_entrypoint_requests_total{code="404"}[5m])) - sum(rate(traefik_service_requests_total{code="404"}[5m]))

# Alerting rules for the essentials
groups:
  - name: traefik
    rules:
      - alert: TraefikConfigReloadFailed
        expr: traefik_config_last_reload_success == 0
        for: 5m
        labels: { severity: page }
      - alert: TraefikBackendDown
        expr: traefik_service_server_up == 0
        for: 2m
        labels: { severity: ticket }
      - alert: TraefikCertificateExpiring
        expr: (traefik_tls_certs_not_after - time()) / 86400 < 14
        labels: { severity: ticket }
      - alert: TraefikHighErrorRatio
        expr: sum by (service) (rate(traefik_service_requests_total{code=~"5.."}[5m])) / sum by (service) (rate(traefik_service_requests_total[5m])) > 0.05
        for: 10m
        labels: { severity: page }

docker logs traefik 2>&1 | grep -iE 'error|level=error' | tail -20     # provider and ACME errors
curl -vk --resolve app.example.com:443:127.0.0.1 https://app.example.com/v1/healthz   # exact SNI and Host, bypassing DNS
curl -s localhost:8080/ping                                               # prints OK when Traefik is healthy (needs ping enabled)

# Routers that are not enabled, with their error
curl -s localhost:8080/api/http/routers | jq -r '.[] | select(.status!="enabled") | [.name, (.error // [] | join("; "))] | @tsv'

# Services with zero servers
curl -s localhost:8080/api/http/services | jq -r '.[] | select((.loadBalancer.servers // []) | length == 0) | .name'

# Routers whose rule mentions a host
curl -s localhost:8080/api/http/routers | jq -r '.[] | select(.rule | test("api.example.com")) | [.name, .priority, .rule] | @tsv'

# Certificate domains stored by the "letsencrypt" resolver
jq -r '.letsencrypt.Certificates[].domain.main' /data/acme.json

# Expiry of the first stored certificate
jq -r '.letsencrypt.Certificates[0].certificate' /data/acme.json | base64 -d | openssl x509 -noout -enddate

# Tail access logs for 5xx only
tail -f /var/log/traefik/access.log | jq -r 'select(.DownstreamStatus >= 500) | [.StartUTC, .RouterName, .DownstreamStatus, .RequestPath] | @tsv'

# Slowest requests in a log file (Duration is in nanoseconds)
jq -r '[.Duration/1000000, .RouterName, .RequestPath] | @tsv' access.log | sort -rn | head

# Requests per router
jq -r '.RouterName' access.log | sort | uniq -c | sort -rn | head

# Traefik labels on a container
docker inspect my-app | jq '.[0].Config.Labels | with_entries(select(.key | startswith("traefik")))'

# Routers by priority, highest first (who wins a conflict)
curl -s localhost:8080/api/http/routers | jq -r '.[] | [.priority // 0, .name, .rule] | @tsv' | sort -rn

# Routers with no TLS on the websecure entrypoint
curl -s localhost:8080/api/http/routers | jq -r '.[] | select((.entryPoints // []) | index("websecure")) | select(.tls == null) | .name'

# Every middleware in use and the routers using it
curl -s localhost:8080/api/http/routers | jq -r '.[] | .name as $r | (.middlewares // [])[] | "\(.)\t\($r)"' | sort

# Middlewares defined but referenced by no router
comm -23 <(curl -s localhost:8080/api/http/middlewares | jq -r '.[].name' | sort) <(curl -s localhost:8080/api/http/routers | jq -r '.[].middlewares[]?' | sort -u)

# Servers currently marked down by health checks
curl -s localhost:8080/api/http/services | jq -r '.[] | .name as $s | (.serverStatus // {}) | to_entries[] | select(.value != "UP") | "\($s)\t\(.key)\t\(.value)"'

# Per-provider object counts and warnings
curl -s localhost:8080/api/overview | jq '{http: .http, tcp: .tcp, providers}'

# Entry points and their addresses
curl -s localhost:8080/api/entrypoints | jq -r '.[] | [.name, .address] | @tsv'

# TLS options and the routers that use each
curl -s localhost:8080/api/http/routers | jq -r '.[] | [(.tls.options // "default"), .name] | @tsv' | sort

# TCP and UDP routers
curl -s localhost:8080/api/tcp/routers | jq -r '.[] | [.name, .rule, .status] | @tsv'; curl -s localhost:8080/api/udp/routers | jq -r '.[] | [.name, .status] | @tsv'

# Full dynamic config, saved for a diff after a change
curl -s localhost:8080/api/rawdata | jq -S . > rawdata-before.json

# Does a request match a router: send it and read the router from the access log
curl -s -o /dev/null -H 'Host: app.example.com' http://127.0.0.1/api/health; tail -1 /var/log/traefik/access.log | jq -r '[.RouterName, .ServiceName, .DownstreamStatus] | @tsv'

# Status code distribution in the last 10,000 log lines
tail -10000 access.log | jq -r .DownstreamStatus | sort | uniq -c | sort -rn

# Top client IPs (ClientHost) in a log file
jq -r .ClientHost access.log | sort | uniq -c | sort -rn | head

# Requests slower than 2 s, by service
jq -r 'select(.Duration > 2000000000) | [.ServiceName, .RequestPath, (.Duration/1e9|tostring)] | @tsv' access.log | sort | uniq -c | sort -rn | head

# Retries per service from the access log (RetryAttempts field)
jq -r 'select(.RetryAttempts > 0) | .ServiceName' access.log | sort | uniq -c

# Certificates stored per resolver with expiry, without printing keys
jq -r '.[] | .Certificates[]? | .domain.main as $d | .certificate | @base64d' /data/acme.json 2>/dev/null | openssl x509 -noout -subject -enddate 2>/dev/null

# Certificate served for a host, straight from Traefik
openssl s_client -connect 127.0.0.1:443 -servername app.example.com </dev/null 2>/dev/null | openssl x509 -noout -subject -issuer -enddate

# Which TLS versions a router accepts
for v in tls1_2 tls1_3; do printf '%s: ' "$v"; openssl s_client -connect 127.0.0.1:443 -servername app.example.com -"$v" </dev/null >/dev/null 2>&1 && echo yes || echo no; done

# Validate a static config file without starting a listener (parse errors print, then Ctrl-C)
traefik --configfile=traefik.yaml --entrypoints.web.address=:0 --log.level=DEBUG 2>&1 | head -20

# Traefik version and Go runtime
traefik version

# Reload dynamic config by touching the directory (file provider with watch: true)
touch /etc/traefik/dynamic/apps.yaml

# Kubernetes: IngressRoutes referencing a Service that does not exist
kubectl get ingressroute -A -o json | jq -r '.items[] | .metadata.namespace as $ns | .metadata.name as $n | .spec.routes[].services[]? | select(.kind == null or .kind == "Service") | "\($ns) \($n) \(.namespace // $ns) \(.name)"' | while read -r ns n sns svc; do kubectl get svc -n "$sns" "$svc" >/dev/null 2>&1 || echo "$ns/$n -> missing $sns/$svc"; done

# Kubernetes: middlewares referenced by IngressRoutes that do not exist
kubectl get ingressroute -A -o json | jq -r '.items[] | .metadata.namespace as $ns | .spec.routes[].middlewares[]? | "\(.namespace // $ns) \(.name)"' | sort -u | while read -r ns m; do kubectl get middleware -n "$ns" "$m" >/dev/null 2>&1 || echo "missing $ns/$m"; done

# Kubernetes: Helm values in force
helm get values traefik -n traefik

#!/usr/bin/env bash
# traefik-health.sh [api-url] [days]: routers in error, servers down, certificates expiring within N days
set -euo pipefail
api=${1:-http://localhost:8080}; days=${2:-14}; rc=0
curl -sf --max-time 5 "$api/ping" >/dev/null || { echo "CRIT: ping failed at $api"; exit 2; }

bad=$(curl -sf "$api/api/http/routers" | jq -r '.[] | select(.status!="enabled") | "  \(.name)\t\(.error // [] | join("; "))"')
[ -z "$bad" ] || { echo "WARN: routers not enabled:"; echo "$bad"; rc=1; }

down=$(curl -sf "$api/api/http/services" | jq -r '.[] | .name as $s | (.serverStatus // {}) | to_entries[] | select(.value!="UP") | "  \($s)\t\(.key)"')
[ -z "$down" ] || { echo "WARN: servers down:"; echo "$down"; rc=1; }

empty=$(curl -sf "$api/api/http/services" | jq -r '.[] | select(.loadBalancer != null and ((.loadBalancer.servers // []) | length == 0)) | "  \(.name)"')
[ -z "$empty" ] || { echo "WARN: services with no servers:"; echo "$empty"; rc=1; }

if [ -r "${ACME_JSON:-/data/acme.json}" ]; then
  now=$(date +%s)
  while IFS=$'\t' read -r domain cert; do
    exp=$(printf '%s' "$cert" | base64 -d | openssl x509 -noout -enddate | cut -d= -f2)
    left=$(( ($(date -d "$exp" +%s) - now) / 86400 ))
    [ "$left" -ge "$days" ] || { echo "WARN: $domain expires in ${left}d"; rc=1; }
  done < <(jq -r '.[] | .Certificates[]? | [.domain.main, .certificate] | @tsv' "${ACME_JSON:-/data/acme.json}")
fi

warnings=$(curl -sf "$api/api/overview" | jq '[.http.routers.warnings, .http.services.warnings, .http.middlewares.warnings] | add // 0')
[ "$warnings" -eq 0 ] || { echo "WARN: $warnings configuration warnings in /api/overview"; rc=1; }
[ $rc -eq 0 ] && echo "OK: $(curl -sf "$api/api/http/routers" | jq length) routers, $(curl -sf "$api/api/http/services" | jq length) services"
exit $rc

#!/usr/bin/env python3
"""Per-service summary of a Traefik JSON access log.

Usage: traefik-log-summary.py access.log [minutes]
Only lines from the last N minutes are counted when given.
"""
import json
import sys
from collections import defaultdict
from datetime import datetime, timedelta, timezone

path = sys.argv[1]
cutoff = datetime.now(timezone.utc) - timedelta(minutes=int(sys.argv[2])) if len(sys.argv) > 2 else None
stats = defaultdict(lambda: {"n": 0, "5xx": 0, "4xx": 0, "durations": []})
with open(path) as f:
    for line in f:
        try:
            e = json.loads(line)
        except json.JSONDecodeError:
            continue
        if cutoff and datetime.fromisoformat(e["StartUTC"].replace("Z", "+00:00")) < cutoff:
            continue
        s = stats[e.get("ServiceName") or e.get("RouterName") or "-"]
        s["n"] += 1
        code = int(e.get("DownstreamStatus", 0))
        s["5xx"] += code >= 500
        s["4xx"] += 400 <= code < 500
        s["durations"].append(e.get("Duration", 0) / 1e6)   # ns -> ms

def pct(xs, p):
    xs = sorted(xs)
    return xs[min(len(xs) - 1, int(p * len(xs)))] if xs else 0

print(f"{'service':40} {'reqs':>8} {'5xx%':>6} {'4xx%':>6} {'p50ms':>8} {'p95ms':>8} {'p99ms':>8}")
for name, s in sorted(stats.items(), key=lambda kv: -kv[1]["n"]):
    d = s["durations"]
    print(f"{name[:40]:40} {s['n']:8d} {100 * s['5xx'] / s['n']:6.2f} {100 * s['4xx'] / s['n']:6.2f} {pct(d, .5):8.1f} {pct(d, .95):8.1f} {pct(d, .99):8.1f}")

#!/usr/bin/env bash
# traefik-user.sh USERS_FILE USERNAME: add or replace one user (password prompted), atomic write, reload check
set -euo pipefail
file=$1; user=$2
[ -f "$file" ] || : > "$file"
tmp=$(mktemp "${file}.XXXXXX"); trap 'rm -f "$tmp"' EXIT
grep -v "^$user:" "$file" > "$tmp" || true                 # drop any existing entry for the user
htpasswd -nB "$user" >> "$tmp"                             # bcrypt; prompts twice, never on the command line
chmod 600 "$tmp"; mv "$tmp" "$file"; trap - EXIT
echo "wrote $(wc -l < "$file") users to $file"
# the basicAuth middleware reads usersFile at each request, so no reload is needed; prove it
if curl -sf -o /dev/null -w '%{http_code}\n' -u "$user:" "${VERIFY_URL:-http://localhost/}" | grep -q '^401$'; then
  echo "middleware answering 401 for an empty password: file in use"
fi