Linux performance

uptime                     # 1, 5, 15 minute load averages: rising or falling
dmesg -T | tail -20        # OOM kills, disk errors, link flaps, hung tasks (may need root)
vmstat 1 5                 # r (runnable), b (blocked), si/so (swapping), us/sy/id/wa/st
mpstat -P ALL 1 3          # one hot CPU hides in the average
pidstat 1 3                # per-process CPU over time, unlike top's snapshot
iostat -xz 1 3             # per device: r/s, w/s, await, aqu-sz, %util
free -h                    # read "available", not "free"
sar -n DEV 1 3             # per-interface rxkB/s and txkB/s against link speed
sar -n TCP,ETCP 1 3        # new connections per second, retransmits per second
top                        # cross-check the picture

cat /proc/pressure/cpu /proc/pressure/io /proc/pressure/memory

some avg10=12.40 avg60=8.10 avg300=3.02 total=160383532
full avg10=0.00 avg60=0.00 avg300=0.00 total=0

mpstat -P ALL 1                  # %usr, %sys, %iowait, %steal, %irq, %soft per CPU
pidstat -u 1                     # per-process %usr and %system
pidstat -w 1                     # voluntary (cswch/s) and involuntary (nvcswch/s) context switches
perf top -F 99                   # live profile by function
perf record -F 99 -a -g -- sleep 30 && perf report --stdio | head -40   # 30 s system-wide with stacks
taskset -cp PID                  # CPU affinity of a process

cat /sys/fs/cgroup/system.slice/my-app.service/cpu.stat   # nr_periods, nr_throttled, throttled_usec
kubectl exec my-pod -- cat /sys/fs/cgroup/cpu.max /sys/fs/cgroup/cpu.stat   # inside a pod

free -h
grep -E 'MemAvailable|Dirty|Writeback|Slab|SUnreclaim|Committed_AS' /proc/meminfo
ps -eo pid,comm,rss,vsz --sort=-rss | head       # resident set size in KiB
smem -rs uss | head                               # unique set size: memory freed if the process exits
slabtop -o | head                                 # kernel object caches (root)
vmstat 1 5                                        # si/so: pages swapped in/out per second
dmesg -T | grep -iE 'out of memory|killed process'

cat /sys/fs/cgroup/system.slice/my-app.service/memory.current
cat /sys/fs/cgroup/system.slice/my-app.service/memory.max
cat /sys/fs/cgroup/system.slice/my-app.service/memory.events   # high, max, oom, oom_kill counters

iostat -xz 1                    # r_await/w_await: ms per I/O including queueing; aqu-sz: average queue length
pidstat -d 1                    # kB_rd/s, kB_wr/s, iodelay per process
iotop -oPa                      # accumulated I/O by process (root)
biolatency 10 1                 # block I/O latency histogram (bcc, root)
df -h; df -i                    # space and inodes
lsof +L1                        # deleted files still held open

sar -n DEV 1                     # rxkB/s, txkB/s, %ifutil per interface
sar -n TCP,ETCP 1                # active/s (outbound), passive/s (inbound), retrans/s
nstat -az | grep -E 'TcpRetransSegs|TcpExtListenDrops|TcpExtListenOverflows|TcpExtTCPSynRetrans'
ss -ti                           # per-socket rtt, cwnd, retrans, delivery rate
ss -ltn 'sport = :8080'          # Recv-Q on a listener = connections waiting for accept()
ip -s link show eth0             # RX/TX errors, dropped, overruns
ethtool -S eth0 | grep -iE 'drop|err|miss|fifo'   # driver-level counters
mtr -rwzbc 100 host.example.com  # per-hop loss and latency

ethtool -g eth0                  # ring buffer sizes, current and maximum
sysctl net.core.somaxconn net.ipv4.tcp_max_syn_backlog

strace -c -f -p PID               # syscall counts and time; Ctrl-C to print the summary
strace -f -tt -e trace=openat,connect -p PID   # specific syscalls with timestamps
execsnoop                         # every exec on the host, with arguments (bcc)
opensnoop -p PID                  # files a process opens (bcc)
tcpconnect                        # outbound TCP connections as they happen (bcc)
biosnoop                          # every block I/O with process and latency (bcc)
funclatency vfs_read              # latency histogram for a kernel function (bcc)
profile -F 99 30                  # sampled stacks for 30 s (bcc)

perf list | head -40                                     # events this kernel and CPU expose
perf stat -- ./my-app --once                             # task-clock, context switches, migrations, page faults, cycles, IPC
perf stat -e task-clock,page-faults,context-switches -p "$PID" -- sleep 10   # counters for a running process
perf stat -a -e cycles,instructions,cache-misses -- sleep 5   # whole machine; IPC well under 1 means memory-bound
perf record -F 99 -g -p "$PID" -- sleep 30               # 99 Hz with stacks for one process
perf record -F 99 -a -g --call-graph dwarf -- sleep 10   # unwind without frame pointers; larger perf.data
perf report --stdio --no-children --percent-limit 1      # self time per symbol, top-down
perf report --sort comm,dso                              # which processes and libraries own the samples
perf annotate --stdio -s my_hot_function                 # instruction-level breakdown of one function
perf script | stackcollapse-perf.pl | flamegraph.pl > flame.svg   # FlameGraph scripts, github.com/brendangregg/FlameGraph
perf trace -p "$PID" -- sleep 5                          # syscalls with durations, via tracepoints rather than ptrace
perf trace -s -p "$PID" -- sleep 5                       # per-syscall summary: count, total, min, avg, max
perf record -e sched:sched_switch -a -- sleep 5 && perf sched latency --sort max   # scheduler run-queue delay per task
perf top -e cache-misses                                 # live view for one event

bpftrace -l 'tracepoint:syscalls:sys_enter_*' | wc -l    # list probes; -lv shows a probe's arguments
bpftrace -lv tracepoint:block:block_rq_issue
bpftrace -e 'tracepoint:syscalls:sys_enter_openat { printf("%-16s %s\n", comm, str(args.filename)); }'
bpftrace -e 'tracepoint:syscalls:sys_enter_* { @[probe] = count(); } interval:s:5 { print(@); clear(@); }'   # syscall rate table every 5 s
bpftrace -e 'kprobe:vfs_read { @bytes[comm] = hist(arg2); }'                    # requested read sizes per command
bpftrace -e 'kretprobe:vfs_read /retval > 0/ { @returned = hist(retval); }'     # actual bytes returned
bpftrace -e 'tracepoint:syscalls:sys_enter_read { @s[tid] = nsecs; }
             tracepoint:syscalls:sys_exit_read /@s[tid]/ { @us[comm] = hist((nsecs - @s[tid]) / 1000); delete(@s[tid]); }'   # read() latency per command
bpftrace -e 'profile:hz:99 /pid == $1/ { @[ustack] = count(); }' "$PID"          # user stacks sampled at 99 Hz, $1 is the first argument
bpftrace -e 'tracepoint:sched:sched_process_exit { printf("%d %s exited\n", pid, comm); }'
bpftrace -e 'kprobe:tcp_retransmit_skb { @[kstack] = count(); }'                # who is retransmitting
bpftrace -e 'uprobe:/usr/lib64/libc.so.6:malloc /pid == $1/ { @sz = hist(arg0); }' "$PID"   # allocation sizes in one process
bpftrace -e 'tracepoint:oom:mark_victim { printf("OOM killed pid %d\n", args.pid); }'

cat /sys/fs/cgroup/cgroup.controllers          # controllers enabled on this host: cpuset cpu io memory pids ...
cat /proc/"$PID"/cgroup                        # 0::/system.slice/my-app.service: the path under /sys/fs/cgroup
systemctl show -p ControlGroup my-app.service
systemd-cgls --no-pager system.slice/my-app.service   # processes in the unit's cgroup
systemd-cgtop -m -d 2 --depth 4                # live table sorted by memory, refreshed every 2 s

cd /sys/fs/cgroup/system.slice/my-app.service
cat cpu.stat          # usage_usec user_usec system_usec nr_periods nr_throttled throttled_usec nr_bursts burst_usec
cat cpu.max           # "max 100000" is unlimited; "50000 100000" is half a CPU per 100 ms period
cat cpu.weight        # 1..10000, default 100; relative share only when CPUs are contended
cat memory.current memory.max memory.high memory.peak     # memory.peak needs kernel 5.19+
grep -E '^(anon|file|kernel|slab|file_dirty|workingset_refault_file|pgscan|pgsteal) ' memory.stat
cat memory.events     # low high max oom oom_kill: counters, not gauges; diff two reads
cat io.stat           # per device: rbytes wbytes rios wios dbytes dios
cat io.pressure memory.pressure cpu.pressure   # PSI scoped to this cgroup
cat pids.current pids.max

sudo systemctl set-property my-app.service CPUQuota=200% MemoryHigh=1500M MemoryMax=2G IOWeight=50
sudo systemctl set-property --runtime my-app.service CPUWeight=500     # experiment first, persist later
systemd-run --scope -p MemoryMax=512M -p CPUQuota=50% -- ./batch-job    # one-off command inside a transient cgroup
systemctl show -p CPUQuotaPerSecUSec,MemoryMax,MemoryHigh my-app.service

sar -B 1 5                       # pgscank/s and pgscand/s: kswapd and direct reclaim scanning; pgsteal/s: pages reclaimed; %vmeff
sar -r 1 3                       # kbmemfree, kbavail, %memused, kbdirty
sar -W 1 3                       # pswpin/s, pswpout/s
grep -E 'workingset_refault|pgmajfault|allocstall' /proc/vmstat   # refaults: cache evicted then read again; allocstall: direct reclaim
watch -n1 'cat /proc/pressure/memory'
swapon --show; zramctl          # swap devices; Fedora uses a zram device by default
sysctl vm.swappiness vm.dirty_ratio vm.dirty_background_ratio vm.min_free_kbytes
cat /sys/kernel/mm/transparent_hugepage/enabled   # [always] causes khugepaged CPU and latency spikes for some workloads; madvise is the safe setting
for p in /proc/[0-9]*; do printf '%s %s %s\n' "$(cat "$p/oom_score" 2>/dev/null)" "${p#/proc/}" "$(cat "$p/comm" 2>/dev/null)"; done | sort -rn | head   # most likely OOM victims

iostat -xz 1                     # r_await w_await d_await f_await (discard, flush); aqu-sz; rareq-sz wareq-sz (average request size)
biolatency -D 10 1               # histogram per disk over 10 s (bcc); -Q includes time queued in the block layer
biosnoop                         # each I/O: time, comm, pid, disk, direction, sector, bytes, latency ms
biotop 5                         # top processes by block I/O
ext4slower 10                    # ext4 operations slower than 10 ms with file name; xfsslower, nfsslower, btrfsslower
fileslower 10                    # synchronous file reads and writes above 10 ms, any filesystem
filetop -C                       # busiest files by reads and writes
cat /proc/"$PID"/io              # rchar wchar read_bytes write_bytes: what this process has done since it started
awk '{print "run=" $1/1e6 "ms wait=" $2/1e6 "ms"}' /proc/"$PID"/schedstat   # run-queue wait time for a task
cat /sys/block/nvme0n1/queue/scheduler         # [none] for NVMe, mq-deadline or bfq for SATA and virtual disks
cat /sys/block/sda/queue/nr_requests /sys/block/sda/queue/rotational
ioping -c 10 /var/lib/postgresql               # request latency to a directory's filesystem, like ping
fio --name=randread --filename=/var/tmp/fio.test --size=1G --rw=randread --bs=4k --iodepth=1 --direct=1 --runtime=20 --time_based --group_reporting   # single-queue random read latency; creates a 1 GiB file
lsblk -o NAME,SIZE,TYPE,MOUNTPOINTS,ROTA,SCHED,RQ-SIZE

# Top 10 processes by resident memory
ps -eo pid,comm,rss --sort=-rss | head -11

# Processes averaging over 1% CPU across 10 seconds
LC_ALL=C pidstat -u 1 10 | awk '/^Average/ && $8 > 1 {print $8, $NF}' | sort -rn | head

# Processes writing to disk now (kB_wr/s)
LC_ALL=C pidstat -d 1 3 | awk '$5 ~ /^[0-9.]+$/ && $5 > 0 {print $5, $NF}' | sort -rn | head

# Established connections by remote address
ss -tn state established | awk 'NR > 1 {sub(/:[^:]*$/, "", $4); print $4}' | sort | uniq -c | sort -rn | head

# Sockets in each TCP state
ss -tan | awk 'NR > 1 {print $1}' | sort | uniq -c | sort -rn

# Largest directories on one filesystem
du -x -d 1 /var 2>/dev/null | sort -h | tail

# Deleted files still consuming space: size, command, path
sudo lsof +L1 | awk 'NR > 1 {print $7, $1, $10}' | sort -rn | head

# Interrupts and context switches per second
vmstat 1 5 | awk 'NR > 2 {print "in=" $11, "cs=" $12}'

# Threads of a process by CPU
top -H -b -n 1 -p "$PID" | head -20

# Live CPU and memory by cgroup
systemd-cgtop -m --depth=3

# Record 20 s of stacks for a flame graph
sudo perf record -F 99 -a -g -- sleep 20 && sudo perf script > out.perf

# PSI for every resource on one line, refreshed each second
watch -n1 'for r in cpu io memory; do printf "%-7s" $r; head -1 /proc/pressure/$r; done'

# Services with the highest memory PSI over the last minute
for f in /sys/fs/cgroup/system.slice/*/memory.pressure; do awk -v u="${f%/memory.pressure}" '/^some/ {split($3,a,"="); print a[2], u}' "$f"; done | sort -rn | head

# Units currently being CPU-throttled (nr_throttled rising is the real test; this shows any throttling since start)
grep -H '^nr_throttled' /sys/fs/cgroup/system.slice/*/cpu.stat | awk -F'[:/ ]' '$NF > 0 {print $NF, $(NF-2)}' | sort -rn | head

# Memory usage per unit, largest first
for d in /sys/fs/cgroup/system.slice/*/; do [ -f "$d/memory.current" ] && printf '%s %s\n' "$(numfmt --to=iec < "$d/memory.current")" "$(basename "$d")"; done | sort -h | tail

# Who was OOM-killed, from the journal, with cgroup path
journalctl -k -g 'oom-kill\|Killed process' --no-pager -n 20

# Kills by systemd-oomd (user-space OOM) in the last day
journalctl -u systemd-oomd --since -1d --no-pager

# Processes in uninterruptible sleep and what they are waiting on
ps -eo pid,stat,wchan:32,comm | awk '$2 ~ /D/'

# Major page faults per process (reads from disk to satisfy a fault)
ps -eo pid,maj_flt,comm --sort=-maj_flt | head

# Per-CPU softirq distribution: one column much higher than the rest means IRQ imbalance
awk '/NET_RX/ {for (i = 2; i <= NF; i++) printf "%s ", $i; print ""}' /proc/softirqs

# IRQs and the CPUs handling them
grep -E 'eth0|nvme' /proc/interrupts | awk '{s = 0; for (i = 2; i < NF - 2; i++) s += $i; print $1, s, $NF}'

# Count syscalls by name for 10 s, no ptrace
sudo bpftrace -e 'tracepoint:raw_syscalls:sys_enter { @[comm] = count(); } interval:s:10 { exit(); }'

# Files being opened right now, by process
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_openat { printf("%-6d %-16s %s\n", pid, comm, str(args.filename)); }'

# Off-CPU time by user stack for one process, 10 s (why it is waiting, not what it is running)
sudo offcputime -p "$PID" -u 10 | head -60

# Block I/O latency histogram per disk for 30 s
sudo biolatency -D 30 1

# fsync latency of a process (database checkpoints, log flushes)
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_fsync /pid == $1/ { @s[tid] = nsecs; } tracepoint:syscalls:sys_exit_fsync /@s[tid]/ { @ms = hist((nsecs - @s[tid]) / 1000000); delete(@s[tid]); }' "$PID"

# Cycles, instructions and IPC for one process over 10 s
sudo perf stat -e cycles,instructions,task-clock -p "$PID" -- sleep 10

# Scheduler run-queue latency: time from wakeup to running, as a histogram
sudo runqlat 10 1

# Hottest kernel and user functions, whole host, 10 s, text output
sudo perf record -F 99 -a -g -- sleep 10 >/dev/null && sudo perf report --stdio --no-children --percent-limit 2 | grep -v '^#' | head -40

# CPU frequency per core (throttled or power-saving cores show low values)
grep MHz /proc/cpuinfo | sort | uniq -c

# Container's own CPU throttling from inside the container
kubectl exec my-pod -- sh -c 'cat /sys/fs/cgroup/cpu.stat | grep -E "nr_throttled|throttled_usec"'

#!/usr/bin/env bash
# usage: cgroup-report.sh [slice]   default system.slice
set -euo pipefail

slice=${1:-system.slice}
base=/sys/fs/cgroup/$slice
[[ -d $base ]] || { echo "no cgroup at $base" >&2; exit 2; }

psi60() { awk '/^some/ {split($3, a, "="); print a[2]}' "$1" 2>/dev/null || echo 0; }
val() { [[ -r $1 ]] && cat "$1" || echo -; }

printf '%-40s %10s %10s %8s %8s %8s\n' UNIT MEM MAX THROTTLED MEMPSI IOPSI
for d in "$base"/*/; do
  unit=$(basename "$d")
  [[ -f $d/memory.current ]] || continue
  mem=$(numfmt --to=iec < "$d/memory.current")
  max=$(val "$d/memory.max"); [[ $max == max ]] || max=$(numfmt --to=iec <<< "$max")
  thr=$(awk '/^nr_throttled/ {print $2}' "$d/cpu.stat" 2>/dev/null || echo -)
  printf '%-40s %10s %10s %8s %8s %8s\n' "$unit" "$mem" "$max" "$thr" "$(psi60 "$d/memory.pressure")" "$(psi60 "$d/io.pressure")"
done | sort -k5 -rn

#!/usr/bin/env bash
# usage: psi-watch.sh [threshold-percent] [interval-seconds]
set -euo pipefail

threshold=${1:-10} interval=${2:-10}
[[ -d /proc/pressure ]] || { echo 'PSI not available (boot with psi=1)' >&2; exit 2; }

snapshot() {
  {
    printf '=== %s: %s avg60=%s%% above %s%%\n' "$(date -Is)" "$1" "$2" "$threshold"
    uptime
    head -1 /proc/pressure/{cpu,io,memory}
    ps -eo pid,pcpu,rss,stat,comm --sort=-pcpu | head -6
    ps -eo pid,pcpu,rss,stat,comm --sort=-rss | head -6
    systemd-cgtop -b -n 1 --depth 2 -m | head -12
  } | logger -t psi-watch
}

while :; do
  for r in cpu io memory; do
    avg60=$(awk '/^some/ {split($3, a, "="); print a[2]}' /proc/pressure/"$r")
    if (( $(printf '%.0f' "$avg60") >= threshold )); then snapshot "$r" "$avg60"; fi
  done
  sleep "$interval"
done

#!/usr/bin/env bash
# usage: perf-bundle.sh my-app.service [seconds]
set -euo pipefail

unit=${1:?unit required} secs=${2:-30}
out=/var/tmp/perf-bundle-${unit%.service}-$(date +%Y%m%dT%H%M%S)
mkdir -p "$out"
cg=$(systemctl show -p ControlGroup --value "$unit")
[[ -n $cg ]] || { echo "$unit has no cgroup (not running?)" >&2; exit 1; }
mainpid=$(systemctl show -p MainPID --value "$unit")

{ uptime; free -h; cat /proc/pressure/{cpu,io,memory}; } > "$out/host.txt"
for f in cpu.stat memory.current memory.max memory.stat memory.events io.stat cpu.pressure memory.pressure io.pressure; do
  cp "/sys/fs/cgroup$cg/$f" "$out/cgroup-$f" 2>/dev/null || true
done
systemctl status --no-pager -l "$unit" > "$out/status.txt" || true
journalctl -u "$unit" -n 500 --no-pager > "$out/journal.txt" || true
ss -tanpi > "$out/sockets.txt"
vmstat 1 "$secs" > "$out/vmstat.txt" &
iostat -xz 1 "$secs" > "$out/iostat.txt" &
pidstat -u -r -d -p "$mainpid" 1 "$secs" > "$out/pidstat.txt" &
perf record -F 99 -g -p "$mainpid" -o "$out/perf.data" -- sleep "$secs" 2> "$out/perf-record.log" || true
wait
perf report -i "$out/perf.data" --stdio --no-children --percent-limit 1 > "$out/perf-report.txt" 2>/dev/null || true
tar -C "$(dirname "$out")" -czf "$out.tar.gz" "$(basename "$out")" && rm -rf "$out"
printf 'bundle: %s.tar.gz\n' "$out"