systemd

systemctl status my-app --no-pager -l
journalctl -u my-app -I --no-pager                          # everything from the latest invocation (v257+)
journalctl -u my-app -b --no-pager -n 50                    # older systemd: last 50 lines this boot
systemctl show my-app -p Result -p ExecMainStatus -p NRestarts
systemd-analyze verify /etc/systemd/system/my-app.service   # syntax, missing binaries, bad dependencies

Result=exit-code
ExecMainStatus=203
NRestarts=4

systemctl list-units --type=service --state=running
systemctl list-unit-files --state=enabled
systemctl cat my-app                          # unit file plus every drop-in, in the order applied
systemctl show my-app -p ExecStart -p User -p MemoryMax
systemctl list-dependencies my-app --reverse  # units that pull this one in
systemctl is-enabled my-app; systemctl is-active my-app   # scriptable: exit status 0 when true
systemd-delta                                 # overridden, extended and masked units system-wide

journalctl -u my-app -f                        # follow
journalctl -u my-app --since '2 hours ago' --until '10 min ago'
journalctl -u my-app -b -1                     # previous boot (needs persistent storage)
journalctl -p warning..err -b                  # priority range
journalctl _PID=1234                           # one process
journalctl -u my-app -o json-pretty -n 1       # every field of the latest record
journalctl -u my-app -g 'timeout|refused'      # regex on MESSAGE; case-insensitive if all lower case
journalctl -k -b                               # kernel messages this boot
journalctl --list-boots
journalctl --disk-usage
journalctl --vacuum-time=7d                    # deletes archived journal files older than 7 days

[Unit]
Description=Ingest worker
Documentation=https://docs.example.com/ingest
After=network-online.target postgresql.service
Wants=network-online.target
StartLimitIntervalSec=300
StartLimitBurst=5

[Service]
Type=notify
User=ingest
Group=ingest
WorkingDirectory=/opt/ingest
EnvironmentFile=-/etc/ingest/env
ExecStart=/opt/ingest/bin/worker --config /etc/ingest/config.yaml
ExecReload=kill -HUP $MAINPID
Restart=on-failure
RestartSec=5
TimeoutStopSec=30

# sandboxing: each line removes access the service does not need
NoNewPrivileges=yes
PrivateTmp=yes
ProtectSystem=strict
ProtectHome=yes
StateDirectory=ingest
ProtectKernelTunables=yes
ProtectControlGroups=yes
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
SystemCallFilter=@system-service
CapabilityBoundingSet=

[Install]
WantedBy=multi-user.target

systemd-analyze security my-app        # exposure score 0 to 10 and the setting behind each point

systemctl edit my-app            # writes /etc/systemd/system/my-app.service.d/override.conf and reloads
systemctl edit --full my-app     # copies the whole unit to /etc/systemd/system/ for larger changes
systemctl revert my-app          # deletes local overrides and drop-ins, back to the vendor unit

# /etc/systemd/system/my-app.service.d/override.conf
[Service]
ExecStart=
ExecStart=/opt/ingest/bin/worker --config /etc/ingest/other.yaml
MemoryMax=2G

systemctl restart my-app         # systemctl edit already ran daemon-reload; the process needs a restart

# /etc/systemd/system/backup.timer
[Unit]
Description=Nightly backup

[Timer]
OnCalendar=*-*-* 02:30:00
RandomizedDelaySec=300
Persistent=true

[Install]
WantedBy=timers.target

systemctl enable --now backup.timer                          # enable the timer, not the service
systemctl list-timers --all                                  # next and last run for each
systemd-analyze calendar --iterations=3 'Mon *-*-* 06:00:00' # validate and show the next elapses
systemctl start backup.service                               # run it now, independent of the timer
journalctl -u backup.service --since today

[Service]
MemoryMax=2G              # hard limit: the OOM killer acts inside the cgroup above this
MemoryHigh=1.5G           # throttle and reclaim above this, before MemoryMax
CPUQuota=150%             # at most 1.5 CPUs of time
CPUWeight=50              # relative share under contention (default 100)
IOWeight=50
TasksMax=512              # processes plus threads

systemd-cgtop                                        # live CPU, memory, I/O per cgroup
systemctl show my-app -p MemoryCurrent -p MemoryPeak -p CPUUsageNSec
systemctl set-property my-app MemoryMax=1G           # apply now and persist as a drop-in
systemd-run --scope -p MemoryMax=1G -p CPUQuota=50% ./heavy-job   # limit an ad-hoc command

systemd-analyze                     # time in firmware, loader, kernel, initrd, userspace
systemd-analyze blame               # units by start time (parallel starts overlap, so not additive)
systemd-analyze critical-chain      # the chain that delayed the default target
systemctl --failed
journalctl -b -p err
systemctl list-jobs                 # jobs still queued when boot hangs

# /etc/systemd/system/my-app.socket
[Socket]
ListenStream=8080                 # TCP on every address; 127.0.0.1:8080 or [::1]:8080 to restrict
ListenStream=/run/my-app/api.sock # AF_UNIX path; both sockets are passed to the same service
SocketMode=0660
SocketUser=my-app
NoDelay=true                      # TCP_NODELAY on accepted connections
FreeBind=true                     # bind an address that is not configured yet at boot

[Install]
WantedBy=sockets.target

# /etc/systemd/system/my-app.service: no [Install] section, the socket starts it
[Service]
Type=exec
User=my-app
ExecStart=/opt/my-app/bin/server --systemd-socket

systemctl enable --now my-app.socket      # enable the socket, not the service
systemctl list-sockets                    # listening address, socket unit and the unit it activates
systemctl status my-app.socket            # "Triggers:" shows the service; "Listen:" the addresses
ss -ltnp 'sport = :8080'                  # the owner is systemd until the service starts
systemctl stop my-app.socket my-app       # stop both; stopping only the service lets the next connection restart it

systemd-socket-activate -l 8080 /opt/my-app/bin/server --systemd-socket   # -l address or port; repeatable
systemd-socket-activate -l 8080 -a /usr/bin/my-handler                    # -a: one instance per connection, like Accept=yes
systemd-socket-activate -l 8080 --inetd /usr/bin/my-handler               # connection on stdin/stdout, inetd style

[Service]
LoadCredential=db-password:/etc/my-app/db-password   # readable only by root on disk
ExecStart=/opt/my-app/bin/server --password-file ${CREDENTIALS_DIRECTORY}/db-password

systemd-analyze security my-app                         # per-directive table; 0 is fully sandboxed, 10 is unrestricted
systemd-analyze security my-app --json=short | jq '[.[] | select(.set == false)] | length'   # settings not applied
systemd-analyze security --threshold=5 my-app           # exit non-zero when exposure is above 5, for CI
systemd-analyze security --offline=yes /etc/systemd/system/my-app.service   # score a file without loading it

systemd-run --pty --wait --uid=my-app -p ProtectSystem=strict -p PrivateTmp=yes \
  -p SystemCallFilter=@system-service -p StateDirectory=my-app /opt/my-app/bin/server --check

mkdir -p ~/.config/systemd/user
systemctl --user daemon-reload
systemctl --user enable --now sync.timer
systemctl --user status sync.service
journalctl --user -u sync.service -f
systemctl --user show-environment           # the environment block user services inherit
systemctl --user import-environment PATH    # copy a variable from this shell into the manager
systemd-run --user --unit=build -p MemoryMax=4G make -j8   # transient user unit with a cgroup limit

sudo loginctl enable-linger alice           # creates /var/lib/systemd/linger/alice; user@1000.service now starts at boot
loginctl show-user alice -p Linger

sudo systemctl --user -M alice@ status sync.timer          # -M USER@ targets that user's manager through the system bus
sudo journalctl _UID="$(id -u alice)" --user-unit=sync.service

# /etc/systemd/journald.conf.d/10-retention.conf
[Journal]
Storage=persistent            # /var/log/journal, created if missing; the default from v259
SystemMaxUse=2G               # cap for /var/log/journal; the smaller of this and SystemKeepFree= wins
SystemKeepFree=5G             # leave this much free for other users of the file system
SystemMaxFileSize=128M        # rotation granularity; vacuuming deletes whole archived files
MaxRetentionSec=1month        # delete entries older than this; default 0 disables age-based deletion
MaxFileSec=1week              # rotate at least weekly so old data is deleted in small steps
RateLimitIntervalSec=30s
RateLimitBurst=10000          # per service; 0 in either disables rate limiting
ForwardToSyslog=no            # yes when rsyslog or a collector reads /run/systemd/journal/syslog
MaxLevelStore=debug           # drop records above this priority before storing
Compress=yes

systemctl restart systemd-journald                # apply; records in flight are kept in /run and re-read
journalctl --disk-usage                           # active plus archived files
journalctl --rotate --vacuum-size=500M            # archive the active files first, then delete the oldest archives
journalctl --vacuum-time=14d                      # by age
journalctl --vacuum-files=10                      # keep at most this many archived files
journalctl --verify                               # checksum every journal file; reports corrupt ones
journalctl --header                               # file headers; useful for out-of-order timestamps after a clock jump
journalctl -N                                     # every field name in use
journalctl -F _SYSTEMD_UNIT                       # every value of one field, for building filters
journalctl --flush                                # /run/log/journal into /var/log/journal once persistent storage exists

[Service]
LogRateLimitIntervalSec=10s
LogRateLimitBurst=200
LogFilterPatterns=~^DEBUG      # drop matching lines before they reach the journal (v253+)
LogExtraFields=TEAM=platform   # add a field to every record for filtering

# Result and exit status of every failed service
systemctl --failed --no-legend --plain | awk '{print $1}' | xargs -r -I{} systemctl show {} -p Id -p Result -p ExecMainStatus

# Services by memory use
systemd-cgtop -m -b -n 1 --depth=2 | head -15

# Which unit owns a process
systemctl status "$(pgrep -o -f worker)"

# Restart count since the unit was last reset
systemctl show my-app -p NRestarts --value

# Processes in a unit's cgroup
systemd-cgls -u my-app.service

# Run a command as the service user in a transient unit and wait for it
systemd-run --uid=ingest --same-dir --wait --pty /opt/ingest/bin/worker --check

# Prevent a unit from being started by anything (symlinks it to /dev/null)
systemctl mask --now my-app

# Journal fields available for a unit
journalctl -u my-app -o verbose -n 1

# Least-sandboxed services first
systemd-analyze security --no-pager | sort -k2 -nr | head

# Units that failed since boot, with the last three journal lines of each
systemctl --failed --no-legend --plain | awk '{print $1}' | while read -r u; do echo "== $u"; journalctl -u "$u" -b -n 3 --no-pager -o cat; done

# Every service that restarted at least once this boot
systemctl list-units --type=service --no-legend --plain | awk '{print $1}' | xargs -r -I{} sh -c 'n=$(systemctl show {} -p NRestarts --value); [ "$n" -gt 0 ] && echo "$n {}"' | sort -rn

# Effective value of one setting after all drop-ins
systemctl show my-app -p ExecStart --value

# Where a unit's files and drop-ins come from
systemctl show my-app -p FragmentPath -p DropInPaths

# Wait until a unit is active before continuing (exit 1 after 60 s)
timeout 60s bash -c 'until systemctl is-active --quiet my-app; do sleep 1; done'

# Time since the unit last became active
systemctl show my-app -p ActiveEnterTimestamp --value

# Sockets systemd is listening on, and the units they trigger
systemctl list-sockets --all

# Timers whose last run failed
systemctl list-timers --all --no-legend --plain | awk '{print $NF}' | xargs -r -I{} sh -c 'systemctl is-failed --quiet {} && echo {}'

# Next three elapses of a calendar expression
systemd-analyze calendar --iterations=3 'Mon..Fri *-*-* 09:00'

# Run a job once, 20 minutes from now, as a transient timer
systemd-run --on-active=20m --unit=cleanup-once /usr/local/bin/cleanup

# Journal lines of a unit as one JSON object per line, for jq
journalctl -u my-app -b -o json | jq -r 'select(.PRIORITY|tonumber <= 3) | .MESSAGE'

# Units logging most this boot
journalctl -b -o json | jq -r '._SYSTEMD_UNIT // empty' | sort | uniq -c | sort -rn | head

# Everything a process wrote, including its children, by cgroup
journalctl -b _SYSTEMD_CGROUP=/system.slice/my-app.service

# Messages the journal suppressed through rate limiting
journalctl -b -g 'Suppressed [0-9]+ messages'

# Kernel OOM kills this boot
journalctl -k -b -g 'Out of memory|oom-kill'

# Follow the journal of a user's service from root
sudo journalctl -f _UID="$(id -u alice)" --user-unit=sync.service

# Unit files that differ from the vendor version
systemd-delta --type=extended,overridden

# Verify every local unit file without starting anything
systemd-analyze verify /etc/systemd/system/*.service

# Run the security scan and fail above a threshold, for CI
systemd-analyze security --threshold=6 my-app --no-pager

# Exposure score for every service, as JSON
for u in $(systemctl list-units --type=service --state=running --no-legend --plain | awk '{print $1}'); do printf '%s ' "$u"; systemd-analyze security "$u" --json=short 2>/dev/null | jq -r 'map(select(.set==false)) | length'; done

# Environment the service will actually see (not your shell's)
systemctl show my-app -p Environment -p EnvironmentFiles

# Memory pressure stall time for a unit's cgroup
cat "/sys/fs/cgroup/system.slice/my-app.service/memory.pressure"

# Set a hard memory limit until the next boot only
systemctl set-property --runtime my-app MemoryMax=512M

# Stop a unit and everything that depends on it
systemctl stop my-app --with-dependencies

# Reboot into the boot menu or a rescue shell
systemctl reboot --boot-loader-menu=10; systemctl rescue

#!/usr/bin/env bash
# usage: failed-units.sh host1 host2 ...   (ssh as a user who can read the journal)
set -euo pipefail

for host in "$@"; do
  printf '== %s\n' "$host"
  ssh -o ConnectTimeout=5 -o BatchMode=yes "$host" bash -s <<'EOF' || printf 'unreachable\n'
set -euo pipefail
units=$(systemctl --failed --no-legend --plain | awk '{print $1}')
[[ -n $units ]] || { echo ok; exit 0; }
for u in $units; do
  systemctl show "$u" -p Id -p Result -p ExecMainStatus -p NRestarts --value | paste -sd ' '
  journalctl -u "$u" -b -n 5 --no-pager -o short-iso | sed 's/^/  /'
done
EOF
done

#!/usr/bin/env bash
# usage: mktimer.sh NAME 'OnCalendar expression' /path/to/command [args...]
set -euo pipefail
name=$1; cal=$2; shift 2
systemd-analyze calendar "$cal" >/dev/null           # fails on a bad expression before anything is written
unit=/etc/systemd/system/$name

cat > "$unit.service" <<EOF
[Unit]
Description=$name job

[Service]
Type=oneshot
ExecStart=$*
DynamicUser=yes
StateDirectory=$name
ProtectSystem=strict
ProtectHome=yes
PrivateTmp=yes
NoNewPrivileges=yes
CapabilityBoundingSet=
SystemCallFilter=@system-service
EOF

cat > "$unit.timer" <<EOF
[Unit]
Description=$name schedule

[Timer]
OnCalendar=$cal
RandomizedDelaySec=120
Persistent=true

[Install]
WantedBy=timers.target
EOF

systemd-analyze verify "$unit.service" "$unit.timer"
systemctl daemon-reload
systemctl enable --now "$name.timer"
systemctl list-timers "$name.timer" --no-pager

#!/usr/bin/env bash
# usage: exposure.sh [threshold]   exits 1 when any service scores above the threshold
set -euo pipefail
threshold=${1:-7}
rc=0
while read -r unit score _; do
  if awk -v s="$score" -v t="$threshold" 'BEGIN { exit !(s > t) }'; then
    printf '%-40s %s\n' "$unit" "$score"
    rc=1
  fi
done < <(systemd-analyze security --no-pager 2>/dev/null | awk 'NR > 1 && $2 ~ /^[0-9.]+$/ {print $1, $2}' | sort -k2 -nr)
exit "$rc"