Cilium

NODE=$(kubectl get pod my-app-7c9d -n my-namespace -o jsonpath='{.spec.nodeName}')
kubectl get pod -n kube-system -l k8s-app=cilium --field-selector spec.nodeName="$NODE" -o name

cilium status --wait                                                       # from outside the cluster
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg status --verbose    # per-node agent view
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg endpoint list       # endpoint ID, identity, labels, policy enforcement
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg identity list       # identity to label mapping
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg bpf ipcache list    # IP to identity (and tunnel endpoint) mapping

kubectl exec -n kube-system <cilium-pod> -- cilium-dbg status | grep -E 'Routing|IPAM|Masquerading'
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg status --all-addresses   # every allocated IP and its owner
kubectl get ciliumnode -o custom-columns='NODE:.metadata.name,CIDR:.spec.ipam.podCIDRs'
kubectl -n kube-system get cm cilium-config -o yaml | grep -E 'routing-mode|tunnel-protocol|ipam|masquerade|kube-proxy-replacement'

kubectl exec -n kube-system <cilium-pod> -- cilium-dbg status | grep KubeProxyReplacement
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg service list    # frontends and their backends
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg bpf lb list     # the same, read from the eBPF map
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg bpf ct list | head   # connection tracking entries

apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata: { name: my-app, namespace: my-namespace }
spec:
  endpointSelector:
    matchLabels: { app: my-app }
  ingress:
    - fromEndpoints: [{ matchLabels: { app: web } }]
      toPorts:
        - ports: [{ port: "8080", protocol: TCP }]
          rules:
            http:
              - { method: "GET", path: "/v1/.*" }      # L7: enforced by Envoy
  egress:
    - toEndpoints: [{ matchLabels: { "k8s:io.kubernetes.pod.namespace": kube-system, "k8s:k8s-app": kube-dns } }]
      toPorts:
        - ports: [{ port: "53", protocol: ANY }]
          rules:
            dns: [{ matchPattern: "*" }]              # routes DNS through the DNS proxy; toFQDNs needs this
    - toFQDNs: [{ matchName: "api.vendor.example.com" }]
      toPorts: [{ ports: [{ port: "443", protocol: TCP }] }]
    - toEntities: ["kube-apiserver"]

kubectl get cnp,ccnp -A
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg policy get                    # all rules the agent holds
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg endpoint get <id> -o json | jq '.[0].status.policy.realized'

apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata: { name: default-deny, namespace: my-namespace }
spec:
  endpointSelector: {}                       # every pod in the namespace
  ingress:
    - fromEntities: [health]                 # keep cilium-health probes working
  egress:
    - toEndpoints:
        - matchLabels: { "k8s:io.kubernetes.pod.namespace": kube-system, "k8s:k8s-app": kube-dns }
      toPorts:
        - ports: [{ port: "53", protocol: ANY }]
          rules: { dns: [{ matchPattern: "*" }] }

  egress:
    - toCIDRSet:
        - cidr: 192.0.2.0/24
          except: [192.0.2.1/32]             # allow the subnet but not the router
    - toEntities: [world]                    # anything not in the cluster; broad, prefer toFQDNs or toCIDR
      toPorts: [{ ports: [{ port: "443", protocol: TCP }] }]
    - icmps:                                 # ICMP is not covered by toPorts
        - fields: [{ type: 8, family: IPv4 }]   # echo request

apiVersion: cilium.io/v2
kind: CiliumClusterwideNetworkPolicy
metadata: { name: block-metadata }
spec:
  endpointSelector: {}
  egressDeny:
    - toCIDR: [169.254.169.254/32]           # cloud metadata endpoint, denied for every pod in the cluster

  ingress:
    - fromEndpoints: [{ matchLabels: { app: frontend } }]
      toPorts:
        - ports: [{ port: "8080", protocol: TCP }]
          rules:
            http:
              - method: GET
                path: "/api/v1/.*"
                headers: ["X-Request-Source: frontend"]   # header must be present with this value
              - method: POST
                path: "/api/v1/orders"

cilium hubble enable --ui                          # enables Hubble, deploys Relay and the UI
cilium hubble port-forward &                       # exposes Relay on localhost:4245
hubble status
hubble observe --pod my-namespace/my-app --last 50
hubble observe --verdict DROPPED --last 100
hubble observe --to-pod my-namespace/my-app --port 8080 -f
hubble observe --protocol dns --last 20
hubble observe --verdict DROPPED -o json | jq -r '.flow | [.source.pod_name, .destination.pod_name, .drop_reason_desc] | @tsv'

hubble observe --since 5m --namespace my-namespace                          # time window instead of --last
hubble observe --from-pod my-namespace/my-app --to-fqdn '*.example.com'     # DNS-resolved destinations
hubble observe --type policy-verdict --verdict DROPPED --since 2m           # policy decisions only
hubble observe --type l7 --http-method POST --http-status 5xx -f            # L7 flows through Envoy
hubble observe --type l7 --protocol dns --since 1m -o json | jq -r 'select(.flow.l7.dns.rcode==3) | .flow.l7.dns.query' | sort | uniq -c   # NXDOMAIN
hubble observe --from-label reserved:world --to-namespace my-namespace      # what is reaching in from outside
hubble observe --identity 16777217 --since 1m                               # by numeric identity (CIDR identities start at 16777216)
hubble observe --node-name ip-10-0-1-23.example.com --verdict DROPPED       # one node's drops
hubble observe --not --to-namespace kube-system --verdict DROPPED           # exclude noise
hubble observe --drop-reason-desc CT_MAP_INSERTION_FAILED --since 10m       # a specific drop reason
hubble observe --to-service my-namespace/my-app --since 1m                  # traffic to a ClusterIP
hubble observe -o compact --since 30s | head                                # shorter than the default output
hubble list nodes                                                            # which agents Relay can reach
hubble list namespaces                                                       # namespaces with flows in the last hour

# During install: identify each cluster
cilium install --set cluster.name=sydney --set cluster.id=1 --context sydney
cilium install --set cluster.name=melbourne --set cluster.id=2 --context melbourne

cilium clustermesh enable --context sydney --service-type LoadBalancer   # deploys clustermesh-apiserver and certs
cilium clustermesh enable --context melbourne --service-type LoadBalancer
cilium clustermesh connect --context sydney --destination-context melbourne   # exchanges CA and endpoints both ways
cilium clustermesh status --context sydney --wait                          # every remote cluster "connected"
cilium connectivity test --context sydney --multi-cluster melbourne        # cross-cluster suite

kubectl exec -n kube-system <cilium-pod> -- cilium-dbg troubleshoot clustermesh     # per-node: DNS, TCP, TLS, etcd to each remote
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg status --verbose | sed -n '/ClusterMesh/,/^$/p'

apiVersion: v1
kind: Service
metadata:
  name: my-app
  namespace: my-namespace
  annotations:
    service.cilium.io/global: "true"      # merge backends from every cluster
    service.cilium.io/shared: "true"      # default; "false" consumes remote backends without exporting local ones
    service.cilium.io/affinity: local     # prefer local backends, fail over to remote; also "remote" or "none"
spec:
  selector: { app: my-app }
  ports: [{ port: 80, targetPort: 8080 }]

POD=my-app-7c9d; NS=my-namespace
NODE=$(kubectl get pod "$POD" -n "$NS" -o jsonpath='{.spec.nodeName}')
AGENT=$(kubectl get pod -n kube-system -l k8s-app=cilium --field-selector spec.nodeName="$NODE" -o name)
EP=$(kubectl exec -n kube-system "$AGENT" -- cilium-dbg endpoint list -o json | jq -r --arg p "$POD" '.[] | select(.status."external-identifiers"."k8s-pod-name"==$p) | .id')

kubectl exec -n kube-system "$AGENT" -- cilium-dbg endpoint get "$EP" -o json | jq '.[0].status | {state, identity: .identity.id, policy: .policy.realized."policy-enabled", labels: .labels."security-relevant"}'
kubectl exec -n kube-system "$AGENT" -- cilium-dbg bpf policy get "$EP"            # the (identity, port, proto) allow map for this endpoint
kubectl exec -n kube-system "$AGENT" -- cilium-dbg bpf ipcache get 10.244.3.17     # identity the datapath believes a remote IP has
kubectl exec -n kube-system "$AGENT" -- cilium-dbg bpf endpoint list               # local IP to endpoint mapping
kubectl exec -n kube-system "$AGENT" -- cilium-dbg bpf tunnel list                 # remote pod CIDR to node IP (tunnel mode)
kubectl exec -n kube-system "$AGENT" -- cilium-dbg bpf nat list | grep 10.244.3.17 # SNAT entries for masqueraded egress
kubectl exec -n kube-system "$AGENT" -- cilium-dbg bpf ct list global | grep 10.244.3.17   # conntrack; flags show direction and state
kubectl exec -n kube-system "$AGENT" -- cilium-dbg monitor -v --related-to "$EP"   # every event touching this endpoint, with L4 detail
kubectl exec -n kube-system "$AGENT" -- cilium-dbg bpf metrics list                # datapath counters: drops and forwards by reason
kubectl exec -n kube-system "$AGENT" -- cilium-dbg debuginfo > "$NODE-debuginfo.txt"   # everything above in one file

cilium status --wait
cilium connectivity test                                           # full suite in namespace cilium-test-1
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg monitor --type drop --type policy-verdict
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg map list --verbose
kubectl logs -n kube-system <cilium-pod> -c cilium-agent --previous | tail -50   # why the agent restarted
cilium sysdump                                                     # zip of logs, maps and policies for a bug report

# Identity, namespace and pod for every endpoint on one node
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg endpoint list -o json | jq -r '.[] | [.status.identity.id, .status."external-identifiers"."k8s-namespace", .status."external-identifiers"."k8s-pod-name"] | @tsv'

# Agents that are not Running
kubectl get pods -n kube-system -l k8s-app=cilium -o wide | grep -v Running

# Drops per source pod in the recent buffer
hubble observe --verdict DROPPED --last 500 -o json | jq -r '.flow.source.pod_name // "unknown"' | sort | uniq -c | sort -rn

# Top talkers by flow count
hubble observe --last 1000 -o json | jq -r '.flow | [.source.pod_name, .destination.pod_name] | @tsv' | sort | uniq -c | sort -rn | head

# Confirm a Service has backends in the datapath (use your ClusterIP)
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg service list | grep -A2 10.96.0.10

# FQDN to IP mappings learned by the DNS proxy
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg fqdn cache list | head -20

# Compare policy revision across agents (should converge to the same number)
kubectl get pods -n kube-system -l k8s-app=cilium -o name | xargs -I{} sh -c 'printf "%s " {}; kubectl exec -n kube-system {} -c cilium-agent -- cilium-dbg policy get -o json | jq .revision'

# Remove connectivity test namespaces and workloads
cilium connectivity test --cleanup

# Agent pod on the node that runs a given pod
kubectl get pod -n kube-system -l k8s-app=cilium --field-selector spec.nodeName="$(kubectl get pod my-app-7c9d -n my-namespace -o jsonpath='{.spec.nodeName}')" -o name

# Endpoint ID for a pod, then its realised policy
EP=$(kubectl exec -n kube-system <cilium-pod> -- cilium-dbg endpoint list -o json | jq -r '.[] | select(.status."external-identifiers"."k8s-pod-name"=="my-app-7c9d") | .id'); kubectl exec -n kube-system <cilium-pod> -- cilium-dbg bpf policy get "$EP"

# Resolve a numeric identity seen in a verdict to labels
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg identity get 16777217

# Endpoints on this node with policy enforcement disabled in either direction
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg endpoint list -o json | jq -r '.[] | select(.status.policy.realized."policy-enabled" != "both") | "\(.id)\t\(.status.policy.realized."policy-enabled")\t\(.status."external-identifiers"."k8s-namespace")/\(.status."external-identifiers"."k8s-pod-name")"'

# Endpoints not in the ready state on this node
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg endpoint list -o json | jq -r '.[] | select(.status.state != "ready") | "\(.id) \(.status.state)"'

# Namespaces with no CiliumNetworkPolicy at all
comm -23 <(kubectl get ns -o name | cut -d/ -f2 | sort) <(kubectl get cnp -A -o jsonpath='{.items[*].metadata.namespace}' | tr ' ' '\n' | sort -u)

# Policies whose endpointSelector matches nothing (candidates for typos)
kubectl get cnp -A -o json | jq -r '.items[] | .metadata.namespace as $ns | "\($ns) \(.metadata.name) " + ((.spec.endpointSelector.matchLabels // {}) | to_entries | map("\(.key)=\(.value)") | join(","))' | while read -r ns name sel; do [ -z "$sel" ] || [ "$(kubectl get pod -n "$ns" -l "$sel" --no-headers 2>/dev/null | wc -l)" -gt 0 ] || echo "$ns/$name selects no pods"; done

# Drop reasons in the last five minutes, counted
hubble observe --verdict DROPPED --since 5m -o json | jq -r .flow.drop_reason_desc | sort | uniq -c | sort -rn

# Identity pairs being denied, resolved to pod labels
hubble observe --type policy-verdict --verdict DROPPED --since 2m -o json | jq -r '.flow | "\(.source.namespace // "world")/\(.source.pod_name // .source.identity) -> \(.destination.namespace // "world")/\(.destination.pod_name // .destination.identity):\(.l4.TCP.destination_port // .l4.UDP.destination_port)"' | sort | uniq -c | sort -rn

# HTTP 5xx seen by Envoy L7 policy, by destination
hubble observe --type l7 --http-status 5xx --since 5m -o json | jq -r '.flow | "\(.destination.pod_name) \(.l7.http.code) \(.l7.http.method) \(.l7.http.url)"' | sort | uniq -c | sort -rn | head

# DNS queries a pod made in the last minute
hubble observe --from-pod my-namespace/my-app --protocol dns --since 1m -o json | jq -r 'select(.flow.l7.type=="REQUEST") | .flow.l7.dns.query' | sort | uniq -c

# Flows from outside the cluster into a namespace
hubble observe --from-label reserved:world --to-namespace my-namespace --since 5m -o compact

# Datapath drop counters on one node, non-zero only
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg bpf metrics list | awk 'NR==1 || ($3+0 > 0 && $1 ~ /Drop|drop/)'

# Conntrack table usage on one node
kubectl exec -n kube-system <cilium-pod> -- sh -c 'cilium-dbg bpf ct list global | wc -l; cilium-dbg map list | grep -E "ct4_global|ct_any4_global"'

# Identity the datapath assigns to a remote IP
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg bpf ipcache get 10.244.3.17

# Nodes as seen by this agent (tunnel endpoints and health)
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg node list

# Cluster-wide health probe matrix from one agent (node and endpoint reachability)
kubectl exec -n kube-system <cilium-pod> -- cilium-health status --probe

# Every Service frontend without a backend
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg service list -o json | jq -r '.[] | select((.status.realized["backend-addresses"] // []) | length == 0) | .status.realized["frontend-address"] | "\(.ip):\(.port)"'

# FQDN policy: names with no cached IPs (rule will block until resolved through the proxy)
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg fqdn cache list -o json | jq -r '.[] | select((.ips // []) | length == 0) | .fqdn'

# Effective Helm values on the running installation
helm get values cilium -n kube-system

# Cilium version per agent (catches a half-finished upgrade)
kubectl get pods -n kube-system -l k8s-app=cilium -o jsonpath='{range .items[*]}{.spec.nodeName}{"\t"}{.spec.containers[?(@.name=="cilium-agent")].image}{"\n"}{end}' | sort -k2 | uniq -c -f1

# Cluster Mesh: connection state to every remote cluster from one agent
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg status --verbose | sed -n '/ClusterMesh/,/^[A-Z]/p'

# Global Services and their affinity settings
kubectl get svc -A -o json | jq -r '.items[] | select(.metadata.annotations["service.cilium.io/global"]=="true") | "\(.metadata.namespace)/\(.metadata.name)\tshared=\(.metadata.annotations["service.cilium.io/shared"] // "true")\taffinity=\(.metadata.annotations["service.cilium.io/affinity"] // "none")"'

# Put an endpoint into policy audit mode (logs verdicts, enforces nothing) and watch
kubectl exec -n kube-system <cilium-pod> -- cilium-dbg endpoint config "$EP" PolicyAuditMode=Enabled && kubectl exec -n kube-system <cilium-pod> -- cilium-dbg monitor -t policy-verdict --related-to "$EP"

#!/usr/bin/env bash
# cilium-node-report.sh: per-node agent health, endpoints, IPAM usage and drop counters
set -euo pipefail
printf '%-40s %-8s %-6s %-14s %-10s %s\n' NODE READY EPS IPAM_USED/MAX DROPS WARNINGS
kubectl get pods -n kube-system -l k8s-app=cilium -o json \
  | jq -r '.items[] | "\(.spec.nodeName) \(.metadata.name) \(.status.containerStatuses[] | select(.name=="cilium-agent") | .ready)"' \
  | while read -r node pod ready; do
    if [ "$ready" != true ]; then printf '%-40s %-8s\n' "$node" NOTREADY; continue; fi
    status=$(kubectl exec -n kube-system "$pod" -c cilium-agent -- cilium-dbg status -o json 2>/dev/null) || { printf '%-40s %-8s\n' "$node" EXEC-FAIL; continue; }
    eps=$(kubectl exec -n kube-system "$pod" -c cilium-agent -- cilium-dbg endpoint list -o json | jq 'length')
    ipam=$(jq -r '.ipam | "\(.allocations | length)/\(((.ipv4 // []) | length) + (.allocations | length))"' <<<"$status")   # used / (used + free)
    drops=$(kubectl exec -n kube-system "$pod" -c cilium-agent -- cilium-dbg bpf metrics list -o json | jq '[.[] | select(.reason | test("Policy|Stale|Unsupported|Invalid")) | .packets] | add // 0')
    warn=$(jq -r '[.controllers[]? | select(.status["consecutive-failure-count"] > 0) | .name] | join(",")' <<<"$status")
    printf '%-40s %-8s %-6s %-14s %-10s %s\n' "$node" ok "$eps" "$ipam" "$drops" "${warn:--}"
  done

#!/usr/bin/env python3
"""Generate a CiliumNetworkPolicy allow-list from observed Hubble flows.

Usage: hubble observe --to-namespace my-namespace --since 30m -o json | ./flows-to-policy.py my-namespace my-app > policy.yaml
Review the output before applying; it reflects what happened, not what should be allowed.
"""
import json
import sys
from collections import defaultdict

import yaml

ns, app = sys.argv[1], sys.argv[2]
pairs: dict[tuple, set] = defaultdict(set)
for line in sys.stdin:
    flow = json.loads(line).get("flow", {})
    dst = flow.get("destination", {})
    if dst.get("namespace") != ns or not any(l == f"k8s:app={app}" for l in dst.get("labels", [])):
        continue
    if flow.get("verdict") not in ("FORWARDED", "DROPPED"):
        continue
    src = flow.get("source", {})
    src_labels = tuple(sorted(l for l in src.get("labels", []) if l.startswith("k8s:app=") or l.startswith("k8s:io.kubernetes.pod.namespace=") or l.startswith("reserved:")))
    l4 = flow.get("l4", {})
    for proto in ("TCP", "UDP"):
        if proto in l4:
            pairs[src_labels].add((str(l4[proto]["destination_port"]), proto))

ingress = []
for labels, ports in sorted(pairs.items()):
    if any(l.startswith("reserved:") for l in labels):
        ingress.append({"fromEntities": [l.split(":", 1)[1] for l in labels], "toPorts": [{"ports": [{"port": p, "protocol": pr} for p, pr in sorted(ports)]}]})
        continue
    match = {l.split("=", 1)[0]: l.split("=", 1)[1] for l in labels}
    ingress.append({"fromEndpoints": [{"matchLabels": match}], "toPorts": [{"ports": [{"port": p, "protocol": pr} for p, pr in sorted(ports)]}]})

policy = {"apiVersion": "cilium.io/v2", "kind": "CiliumNetworkPolicy",
          "metadata": {"name": f"{app}-observed", "namespace": ns},
          "spec": {"endpointSelector": {"matchLabels": {"app": app}}, "ingress": ingress}}
yaml.safe_dump(policy, sys.stdout, sort_keys=False)