HTTP and curl

# Handshake, request headers (>), response headers (<), status
curl -sSv https://api.example.com/health -o /dev/null

# Where the time goes
curl -o /dev/null -sw 'code=%{http_code} dns=%{time_namelookup} conn=%{time_connect} tls=%{time_appconnect} ttfb=%{time_starttransfer} total=%{time_total}\n' https://api.example.com/health

# Bypass DNS and hit one backend, keeping the Host header and TLS SNI
curl -sv --resolve api.example.com:443:192.0.2.10 https://api.example.com/health -o /dev/null

cat > curl-format.txt <<'EOF'
     dns  %{time_namelookup}s
 connect  %{time_connect}s
     tls  %{time_appconnect}s
 waiting  %{time_starttransfer}s
   total  %{time_total}s
    size  %{size_download} bytes
    code  %{http_code}
 version  %{http_version}
EOF

curl -o /dev/null -s -w '@curl-format.txt' https://api.example.com/health

     dns  0.012345s
 connect  0.030112s
     tls  0.071870s
 waiting  0.180553s
   total  0.181204s
    size  15 bytes
    code  200
 version  2

# Ten samples, sorted, to separate a slow server from a slow tail
for i in $(seq 10); do curl -o /dev/null -sw '%{time_total}\n' https://api.example.com/health; done | sort -n

# JSON body; -H is needed because -d defaults to form encoding
curl -X POST https://api.example.com/items \
  -H 'Content-Type: application/json' \
  -d '{"name":"widget","qty":4}'

# JSON from a file with --json (sets Content-Type and Accept)
curl --json @payload.json https://api.example.com/items

# Multipart form upload
curl -F 'file=@report.csv' -F 'note=monthly' https://api.example.com/upload

# URL-encoded query string
curl -G https://api.example.com/search --data-urlencode 'q=name with spaces' --data-urlencode 'limit=10'

# PATCH with a merge patch
curl -X PATCH -H 'Content-Type: application/merge-patch+json' -d '{"qty":5}' https://api.example.com/items/1

# PUT a file
curl -T ./artifact.tgz https://uploads.example.com/path/

# Negotiated protocol, cipher, certificate subject, issuer and expiry
curl -svI https://example.com 2>&1 | grep -E 'SSL connection|subject:|issuer:|expire date'

# Force a TLS version range
curl --tlsv1.3 -sI https://example.com
curl --tlsv1.2 --tls-max 1.2 -sI https://example.com

# Trust a private CA for this call
curl --cacert ca.pem https://internal.example.com

# Client certificate (mTLS)
curl --cert client.pem --key client.key https://mtls.example.com

curl -x http://proxy.example.com:3128 https://api.example.com
export https_proxy=http://proxy.example.com:3128 no_proxy=.example.com,localhost,10.0.0.0/8
curl --proxy-user "alice:$PROXY_PASSWORD" -x http://proxy.example.com:3128 https://api.example.com
curl -x socks5h://localhost:1080 https://api.example.com   # socks5h: the proxy resolves DNS
curl -sv https://api.example.com -o /dev/null 2>&1 | grep -iE 'CONNECT|proxy'

# Every hop with its status and Location
curl -sIL https://example.com | grep -iE '^HTTP/|^location:'

# Final URL after all redirects
curl -o /dev/null -sw '%{url_effective}\n' -L https://example.com

# See what a cache would do with this response
curl -sSI https://example.com/app.js | grep -iE '^(cache-control|etag|last-modified|age|vary|expires|x-cache):'

# Conditional GET: 304 if the ETag still matches
curl -so /dev/null -w '%{http_code}\n' -H 'If-None-Match: "5d8c72a5edda8d6a"' https://example.com/app.js

# Download again only when the ETag changed (7.68.0 and later)
curl -sS --etag-compare etag.txt --etag-save etag.txt -o app.js https://example.com/app.js

# Reproduce a preflight; the server must return 2xx and the Access-Control-* headers
curl -sS -X OPTIONS https://api.example.com/items \
  -H 'Origin: https://app.example.com' \
  -H 'Access-Control-Request-Method: PUT' \
  -H 'Access-Control-Request-Headers: authorization,content-type' \
  -D - -o /dev/null | grep -iE '^(HTTP/|access-control-)'

# Credentials from ~/.netrc (mode 0600, one line: machine api.example.com login alice password ...)
curl -n https://api.example.com/me                     # --netrc-file for another path

# OAuth2 client-credentials flow, then use the token
token=$(curl -fsS -u "$CLIENT_ID:$CLIENT_SECRET" -d 'grant_type=client_credentials' https://auth.example.com/oauth/token | jq -r .access_token)
curl -fsS --oauth2-bearer "$token" https://api.example.com/items

# Which version was negotiated
curl -so /dev/null -w '%{http_version}\n' https://example.com

# ALPN offer and answer
curl -sv https://example.com -o /dev/null 2>&1 | grep -iE 'ALPN|using HTTP'

# HTTP/2 without TLS, for a backend behind a terminating proxy (no upgrade, straight h2c)
curl --http2-prior-knowledge http://localhost:8080/health

# Try HTTP/3 and fall back, or insist on it
curl --http3 -sI https://example.com
curl --http3-only -so /dev/null -w '%{http_version}\n' https://example.com

# Log in, keep the session cookie, use it
curl -sS -c jar.txt -d "username=alice&password=$PASSWORD" https://app.example.com/login -o /dev/null
curl -sS -b jar.txt https://app.example.com/dashboard

# Send a literal cookie without a file
curl -b 'session=abc123; theme=dark' https://app.example.com/


# What the server set, including attributes
curl -sSI https://app.example.com/login | grep -i '^set-cookie'

if ! body=$(curl -fsS --max-time 10 --retry 3 --retry-connrefused https://api.example.com/health); then
  printf 'health check failed\n' >&2
  exit 1
fi

tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT
code=$(curl -sS -o "$tmp/body" -D "$tmp/headers" -w '%{http_code}' --max-time 30 \
  --json @payload.json https://api.example.com/items)
case $code in
  2*)  ;;
  429) sleep "$(awk 'tolower($1) == "retry-after:" {print $2}' "$tmp/headers" | tr -d '\r')" ;;
  *)   printf 'unexpected %s: %s\n' "$code" "$(cat "$tmp/body")" >&2; exit 1 ;;
esac

curl -sS -o /dev/null -w '%{json}\n' --retry 3 --max-time 10 https://api.example.com/health \
  | jq '{code: .http_code, total: .time_total, retries: .num_retries, ip: .remote_ip, err: .errormsg}'

# Response headers of a GET, sorted
curl -sS -D - -o /dev/null https://example.com | sort

# Poll until a service is healthy
until curl -fsS --max-time 2 http://localhost:8080/health >/dev/null; do sleep 1; done

# 50 requests, 10 at a time, count the status codes
seq 50 | xargs -P10 -I{} curl -o /dev/null -sw '%{http_code}\n' https://api.example.com/ | sort | uniq -c

# Compare JSON config between two environments (needs jq)
diff <(curl -s https://staging.example.com/api/config | jq -S .) <(curl -s https://prod.example.com/api/config | jq -S .)

# Resume a partial download
curl -C - -O https://example.com/big.iso

# Send an HMAC-signed webhook; the secret comes from the environment
body='{"event":"test"}'; sig=$(printf '%s' "$body" | openssl dgst -sha256 -hmac "$WEBHOOK_SECRET" -hex | awk '{print $NF}'); curl -X POST -H "X-Signature: sha256=$sig" -H 'Content-Type: application/json' --data-binary "$body" https://api.example.com/hook

# Negotiated HTTP version and scheme
curl -so /dev/null -w '%{http_version} %{scheme}\n' https://example.com

# Plain HTTP virtual host test against an IP
curl -H 'Host: api.example.com' http://192.0.2.10/health

# Slowest of several endpoints
for u in "${urls[@]}"; do printf '%s %s\n' "$(curl -o /dev/null -sw '%{time_total}' "$u")" "$u"; done | sort -rn | head

# Every write-out variable as JSON, for a log line
curl -sS -o /dev/null -w '%{json}\n' https://api.example.com/health

# Response headers as JSON (7.83.0 and later)
curl -sS -o /dev/null -w '%{header_json}\n' https://example.com | jq '.["cache-control"]'

# Which IP and port curl actually talked to (after DNS and redirects)
curl -so /dev/null -w '%{remote_ip}:%{remote_port} %{http_code}\n' -L https://example.com





# Talk to a service on a Unix socket (Docker, Podman, systemd-style daemons)
curl -sS --unix-socket /var/run/docker.sock http://localhost/version






# HEAD every URL in a file, 10 at a time, print code and URL
xargs -a urls.txt -P10 -I{} curl -so /dev/null -w '%{http_code} {}\n' -I {}





# Bearer token from the environment without putting it on the command line (8.3.0 and later)
curl --variable %API_TOKEN --expand-header 'Authorization: Bearer {{API_TOKEN}}' https://api.example.com/me

#!/usr/bin/env bash
# usage: http-latency.sh URL [samples]
set -euo pipefail
url=${1:?url required}; n=${2:-20}
tmp=$(mktemp); trap 'rm -f "$tmp"' EXIT
for _ in $(seq "$n"); do
  curl -sS -o /dev/null --max-time 15 \
    -w '%{time_namelookup} %{time_connect} %{time_appconnect} %{time_starttransfer} %{time_total} %{http_code}\n' \
    "$url" >>"$tmp" || true
done
printf '%-8s %8s %8s %8s\n' phase p50 p90 max
for col in 1:dns 2:tcp 3:tls 4:ttfb 5:total; do
  i=${col%%:*}; name=${col#*:}
  sort -k"$i" -n "$tmp" | awk -v i="$i" -v name="$name" \
    '{v[NR]=$i} END {printf "%-8s %8.3f %8.3f %8.3f\n", name, v[int(NR*0.5)+1], v[int(NR*0.9)+1], v[NR]}'
done
printf 'codes: '; awk '{print $6}' "$tmp" | sort | uniq -c | tr '\n' ' '; echo

#!/usr/bin/env bash
# usage: http-check.sh urls.txt   (one URL per line, # comments allowed)
set -euo pipefail
list=${1:?url list required}; max_ms=${MAX_MS:-2000}; min_days=${MIN_CERT_DAYS:-14}
rc=0
while IFS= read -r url; do
  [[ -z $url || $url == \#* ]] && continue
  out=$(curl -sS -o /dev/null --max-time 10 -w '%{http_code} %{time_total} %{exitcode} %{errormsg}' "$url" || true)
  read -r code total exit_code msg <<<"$out"
  ms=$(awk -v t="$total" 'BEGIN {printf "%d", t * 1000}')
  if [[ $exit_code != 0 ]]; then printf 'FAIL %s curl(%s): %s\n' "$url" "$exit_code" "$msg"; rc=1; continue; fi
  [[ $code == 2* || $code == 3* ]] || { printf 'FAIL %s status %s\n' "$url" "$code"; rc=1; }
  (( ms <= max_ms )) || { printf 'SLOW %s %sms\n' "$url" "$ms"; rc=1; }
  if [[ $url == https://* ]]; then
    host=${url#https://}; host=${host%%/*}; host=${host%%:*}
    exp=$(echo | openssl s_client -servername "$host" -connect "$host:443" 2>/dev/null | openssl x509 -noout -enddate | cut -d= -f2)
    days=$(( ($(date -d "$exp" +%s) - $(date +%s)) / 86400 ))
    (( days >= min_days )) || { printf 'CERT %s expires in %d days\n' "$url" "$days"; rc=1; }
  fi
done <"$list"
exit "$rc"