Nginx

/etc/nginx/nginx.conf            # main, events, http; includes the rest
/etc/nginx/conf.d/*.conf         # one server block per file (Debian also uses sites-available and sites-enabled)
/etc/nginx/snippets/*.conf       # reusable fragments pulled in with include
/etc/nginx/mime.types
/var/log/nginx/{access,error}.log
/var/cache/nginx/                # proxy_cache_path and temp files

user  nginx;
worker_processes  auto;            # one per CPU
error_log  /var/log/nginx/error.log warn;
pid        /run/nginx.pid;

events {
    worker_connections  4096;      # per worker; each proxied request uses two
}

http {
    include       mime.types;
    default_type  application/octet-stream;
    sendfile      on;
    tcp_nopush    on;
    keepalive_timeout  65;
    server_tokens off;             # drop the version from headers and error pages
    include /etc/nginx/conf.d/*.conf;
}

server {                                   # catch-all: refuse unknown names instead of serving the first vhost
    listen 80 default_server;
    listen 443 ssl default_server;
    ssl_reject_handshake on;               # 1.19.4+: abort TLS for names you do not serve
    return 444;                            # close the connection without a response
}

server {
    listen 443 ssl;
    http2 on;
    server_name example.com www.example.com;
    ...
}

location = / { ... }                       # only the root URI
location / { ... }                         # everything not matched more specifically
location ^~ /static/ { ... }               # static files, and no regex gets a look-in
location ~* \.(png|jpg|css|js)$ { ... }    # any other path with these extensions
location /api/ { ... }                     # would lose to the regex above for /api/logo.png

map $http_upgrade $connection_upgrade {    # in the http context
    default upgrade;
    ''      close;
}

upstream my-app {
    server 127.0.0.1:8080;
    keepalive 32;                          # idle connections to keep per worker
}

server {
    listen 443 ssl;
    http2 on;
    server_name app.example.com;

    location / {
        proxy_pass http://my-app;
        proxy_http_version 1.1;
        proxy_set_header Host              $host;
        proxy_set_header X-Real-IP         $remote_addr;
        proxy_set_header X-Forwarded-For   $proxy_add_x_forwarded_for;   # appends to an existing header
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header X-Forwarded-Host  $host;
        proxy_set_header Upgrade           $http_upgrade;               # WebSocket
        proxy_set_header Connection        $connection_upgrade;         # "upgrade" for WS, "" (keepalive) otherwise
        proxy_read_timeout  300s;          # idle WebSocket connections are closed after this
        proxy_send_timeout  300s;
        proxy_connect_timeout 5s;
    }

    location /events/ {                    # server-sent events or streaming responses
        proxy_pass http://my-app;
        proxy_http_version 1.1;
        proxy_set_header Connection "";
        proxy_buffering off;               # deliver each chunk as it arrives
        proxy_read_timeout 1h;
    }
}

location /api/ { proxy_pass http://my-app; }        # /api/users -> /api/users
location /api/ { proxy_pass http://my-app/; }       # /api/users -> /users (the matched prefix is replaced)
location /api/ { proxy_pass http://my-app/v2/; }    # /api/users -> /v2/users
location ~ ^/api/ { proxy_pass http://my-app/; }    # error: a URI part is not allowed with a regex location

resolver 127.0.0.11 valid=30s ipv6=off;    # Docker's embedded DNS; use the host resolver elsewhere
set $backend http://my-app:8080;
proxy_pass $backend$request_uri;           # with a variable, Nginx no longer rewrites the URI for you

server {
    listen 443 ssl;
    listen [::]:443 ssl;
    listen 443 quic reuseport;             # HTTP/3; reuseport is required with several workers
    listen [::]:443 quic reuseport;
    http2 on;
    http3 on;
    server_name example.com;

    ssl_certificate     /etc/letsencrypt/live/example.com/fullchain.pem;   # leaf plus intermediates
    ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem;
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305;
    ssl_prefer_server_ciphers off;         # let modern clients choose; the list above has no weak entries
    ssl_ecdh_curve X25519:prime256v1:secp384r1;
    ssl_session_timeout 1d;
    ssl_session_cache shared:SSL:10m;      # about 40k sessions
    ssl_session_tickets off;

    add_header Alt-Svc 'h3=":443"; ma=86400';                 # tell browsers HTTP/3 is available
    add_header Strict-Transport-Security "max-age=63072000" always;

    location / { proxy_pass http://my-app; ... }
}

server {
    listen 80;
    listen [::]:80;
    server_name example.com;
    location /.well-known/acme-challenge/ { root /var/www/acme; }   # keep HTTP-01 working
    location / { return 301 https://$host$request_uri; }
}

server {
    listen 443 ssl;
    http2 on;
    server_name www.example.com;
    root /srv/www/example.com;             # request /docs/a.html -> /srv/www/example.com/docs/a.html
    index index.html;

    location / {
        try_files $uri $uri/ /index.html;  # single-page app: unknown paths get the shell
    }

    location /downloads/ {
        alias /srv/files/;                 # request /downloads/a.zip -> /srv/files/a.zip; trailing slashes must match
    }

    location ~* \.(css|js|woff2|png|svg)$ {
        expires 1y;
        add_header Cache-Control "public, immutable";
        access_log off;
    }

    gzip on;
    gzip_types text/plain text/css application/json application/javascript image/svg+xml;
    gzip_min_length 1024;
}

proxy_cache_path /var/cache/nginx/app levels=1:2 keys_zone=app:10m max_size=2g inactive=60m use_temp_path=off;

location /api/catalogue/ {
    proxy_pass http://my-app;
    proxy_cache app;
    proxy_cache_key $scheme$host$request_uri;
    proxy_cache_valid 200 301 10m;         # only when the upstream sends no Cache-Control
    proxy_cache_valid 404 1m;
    proxy_cache_use_stale error timeout updating http_502 http_503;   # serve stale while the backend is down
    proxy_cache_lock on;                   # one request fills the cache; others wait
    proxy_cache_bypass $http_cache_control;
    add_header X-Cache-Status $upstream_cache_status;   # HIT, MISS, EXPIRED, STALE, BYPASS
}

limit_req_zone  $binary_remote_addr zone=per_ip:10m rate=10r/s;   # 10m holds about 160k addresses
limit_conn_zone $binary_remote_addr zone=conn_per_ip:10m;

server {
    location /api/ {
        limit_req zone=per_ip burst=20 nodelay;   # allow bursts of 20, reject beyond that immediately
        limit_req_status 429;                      # default is 503
        limit_conn conn_per_ip 20;
        proxy_pass http://my-app;
    }

    location /admin/ {
        allow 192.0.2.0/24;
        deny  all;                                 # evaluated in order; first match wins
        auth_basic           "Restricted";
        auth_basic_user_file /etc/nginx/.htpasswd; # htpasswd -B for bcrypt; readable by the worker user
        proxy_pass http://my-app;
    }

    location /internal/ { internal; }              # only reachable via rewrite, error_page or X-Accel-Redirect
}

return 301 https://$host$request_uri;                     # cheapest redirect; no regex engine
return 302 /maintenance.html;
rewrite ^/old/(.*)$ /new/$1 permanent;                     # regex redirect (301); "redirect" gives 302
rewrite ^/blog/(\d+)$ /posts?id=$1 break;                  # internal rewrite; stop processing, stay in this location
rewrite ^/blog/(\d+)$ /posts?id=$1 last;                   # internal rewrite; re-run location matching
location = /health { return 200 'ok'; default_type text/plain; }
absolute_redirect off;                                     # emit relative Location headers behind a proxy

upstream my-app {
    least_conn;                            # default is weighted round-robin; also ip_hash, hash $key [consistent], random
    zone my-app 64k;                       # shared state across workers so failure counts are cluster-wide
    server 10.0.0.11:8080 weight=2 max_fails=3 fail_timeout=30s;
    server 10.0.0.12:8080 max_fails=3 fail_timeout=30s;
    server 10.0.0.13:8080 backup;          # used only when every primary is marked failed
    server 10.0.0.14:8080 down;            # kept for ip_hash stability, never used
    keepalive 64;
}

location / {
    proxy_pass http://my-app;
    proxy_next_upstream error timeout http_502 http_503;   # retry another server on these; not for POST unless idempotent
    proxy_next_upstream_tries 2;
    proxy_next_upstream_timeout 10s;
}

log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                '$status $body_bytes_sent "$http_referer" "$http_user_agent" '
                'rt=$request_time urt=$upstream_response_time ua=$upstream_addr us=$upstream_status '
                'cs=$upstream_cache_status h=$host';
log_format json escape=json '{"ts":"$time_iso8601","host":"$host","remote":"$remote_addr","method":"$request_method","uri":"$request_uri","status":$status,"bytes":$body_bytes_sent,"rt":$request_time,"urt":"$upstream_response_time","ua":"$upstream_addr","ref":"$http_referer","agent":"$http_user_agent"}';

access_log /var/log/nginx/access.log json buffer=64k flush=5s;   # buffered: lines appear up to 5s late
error_log  /var/log/nginx/error.log warn;                         # debug needs --with-debug; info is enough to see upstream retries

journalctl -u nginx --since -1h                   # startup and reload failures land here, not in error.log
tail -f /var/log/nginx/error.log | grep -v 'No such file'
awk '$9 >= 500' /var/log/nginx/access.log | tail  # combined format; column 9 is $status

# Validate and reload only if valid
nginx -t && systemctl reload nginx

# Effective configuration with includes expanded, comments stripped
nginx -T 2>/dev/null | grep -Ev '^\s*(#|$)'

# Every server_name and its listen ports
nginx -T 2>/dev/null | grep -E '^\s*(server_name|listen)' | sed 's/^\s*//'

# Which config file defines a location
grep -rn 'location /api/' /etc/nginx/

# Modules compiled in
nginx -V 2>&1 | tr ' ' '\n' | grep -- '--with'

# Confirm HTTP/2 and HTTP/3 negotiate
curl -sI --http2 https://example.com/ | head -1; curl -sI --http3 https://example.com/ | head -1

# Requests per minute over the last hour (combined format)
awk -v d="$(date -d '-1 hour' '+%d/%b/%Y:%H')" '$4 ~ d {print substr($4,2,17)}' /var/log/nginx/access.log | sort | uniq -c

# Top 20 client addresses
awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head -20

# Top URIs returning 5xx
awk '$9 ~ /^5/ {print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head

# Status code distribution for one path
grep '"GET /api/' /var/log/nginx/access.log | awk '{print $9}' | sort | uniq -c

# Upstream errors by reason, from the error log
grep -oE '(upstream [a-z ]+|connect\(\) failed[^,]*|no live upstreams)' /var/log/nginx/error.log | sort | uniq -c | sort -rn

# Bytes served per host today (json format with jq)
jq -r 'select(.ts | startswith("'"$(date +%F)"'")) | "\(.host) \(.bytes)"' /var/log/nginx/access.log | awk '{s[$1]+=$2} END {for (h in s) print s[h], h}' | sort -rn

# Live worker connections (stub_status on 127.0.0.1)
curl -s http://127.0.0.1/nginx_status

# Established connections per upstream
ss -tn state established '( dport = :8080 )' | tail -n +2 | wc -l

# Certificate expiry of every ssl_certificate in the config
nginx -T 2>/dev/null | awk '$1=="ssl_certificate" {gsub(";","",$2); print $2}' | sort -u | xargs -I{} sh -c 'printf "%s " {}; openssl x509 -enddate -noout -in {}'

# Hash a password for auth_basic without the apache2-utils package
openssl passwd -apr1 >> /etc/nginx/.htpasswd   # prompts; prepend "alice:" to the line afterwards

# Rotate logs without a restart (reopen after moving the files)
mv /var/log/nginx/access.log /var/log/nginx/access.log.1 && nginx -s reopen

# Trace what a worker is doing right now
strace -p "$(pgrep -f 'nginx: worker' | head -1)" -f -e trace=network -s 80

# Purge one URL from the proxy cache: the file name is the MD5 of proxy_cache_key ($scheme$host$request_uri above)
find /var/cache/nginx/app -type f -name "$(printf 'httpsexample.com/api/catalogue/' | md5sum | cut -c1-32)" -delete

#!/usr/bin/env bash
set -euo pipefail
host=${1:?hostname}
cert=/etc/letsencrypt/live/$host/fullchain.pem
served=$(openssl s_client -connect 127.0.0.1:443 -servername "$host" </dev/null 2>/dev/null \
  | openssl x509 -noout -fingerprint -sha256)
disk=$(openssl x509 -noout -fingerprint -sha256 -in "$cert")
if [[ "$served" == "$disk" ]]; then
  printf '%s: certificate already live\n' "$host"
  exit 0
fi
nginx -t
systemctl reload nginx
printf '%s: reloaded\n' "$host"

#!/usr/bin/env python3
import json, sys, collections, statistics
n = int(sys.argv[1]) if len(sys.argv) > 1 else 10000
lines = collections.deque(open("/var/log/nginx/access.log"), maxlen=n)
by_prefix = collections.defaultdict(list)
status = collections.Counter()
for line in lines:
    try:
        r = json.loads(line)
    except ValueError:
        continue
    prefix = "/" + r["uri"].split("/")[1].split("?")[0]
    by_prefix[prefix].append(float(r["rt"]))
    status[f"{prefix} {str(r['status'])[0]}xx"] += 1
for prefix, times in sorted(by_prefix.items(), key=lambda kv: -len(kv[1])):
    times.sort()
    p95 = times[int(len(times) * 0.95) - 1] if len(times) > 1 else times[0]
    errs = status.get(f"{prefix} 5xx", 0)
    print(f"{prefix:30} n={len(times):6} p50={statistics.median(times):.3f}s p95={p95:.3f}s 5xx={errs}")

#!/usr/bin/env bash
set -euo pipefail
path=${1:-/healthz}
rc=0
while read -r addr; do
  code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 "http://$addr$path" || echo 000)
  if [[ $code == 2* ]]; then
    printf 'ok   %s %s\n' "$addr" "$code"
  else
    printf 'FAIL %s %s\n' "$addr" "$code" >&2
    rc=1
  fi
done < <(nginx -T 2>/dev/null | awk '/^upstream/,/}/ {if ($1=="server") {gsub(";","",$2); print $2}}' | grep -v ':unix' | sort -u)
exit "$rc"