diff --git a/VERSION b/VERSION index 1e8b314..7f8f011 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6 +7 diff --git a/bin/dejima-devbox-audit b/bin/dejima-devbox-audit index 8d32884..82dfe7a 100755 --- a/bin/dejima-devbox-audit +++ b/bin/dejima-devbox-audit @@ -679,7 +679,35 @@ drift_state() { # send queue. A flag added without moving the field would have recorded a # queue depth as a port, quietly, on every line. if command -v ss >/dev/null 2>&1; then - ss -H -ltun 2>/dev/null | awk 'NF {print "PORT " $1 " " $5}' + # EPHEMERAL PORTS ARE NOT LISTENERS, and including them made this state + # differ from itself every few minutes. + # + # `ss -l` with -u reports UDP sockets in UNCONN, which is what a client + # socket looks like: a resolver doing a lookup holds one on a random + # high port for as long as the query takes. TCP has the same shape for + # anything that binds before connecting. So a box doing nothing produced + # a new PORT line every cycle and lost the one from the cycle before, + # which a watcher reads as a listener appearing and disappearing - a max + # priority alert every half hour on a machine where nothing happened, + # and a baseline that cannot be accepted because the next sample differs + # again. + # + # The range comes from the kernel rather than being written down, since + # it is tunable and a wrong constant here would either keep the flapping + # or hide real services. + # + # What this gives up: a listener the kernel would also hand out as an + # ephemeral port is no longer recorded. That is a real loss, and it is + # the better trade - the alternative is an alert every cycle, and an + # alert that fires constantly is one nobody reads. + local ep_lo ep_hi + read -r ep_lo ep_hi < /proc/sys/net/ipv4/ip_local_port_range 2>/dev/null || true + : "${ep_lo:=32768}" "${ep_hi:=60999}" + ss -H -ltun 2>/dev/null | awk -v lo="$ep_lo" -v hi="$ep_hi" ' + NF { + n = split($5, a, ":"); port = a[n] + 0 + if (port < lo || port > hi) print "PORT " $1 " " $5 + }' fi return 0 } diff --git a/test/addresses.bats b/test/addresses.bats index a5528b2..3b7db39 100644 --- a/test/addresses.bats +++ b/test/addresses.bats @@ -43,6 +43,8 @@ setup() { allowed=( 0.0.0.0 # wildcard bind, sshd's ListenAddress 127.0.0.1 # loopback, as a rejected upstream resolver + 127.0.0.11 # Docker's embedded resolver, in the drift test that + # shows an ephemeral client socket is not a listener 127.0.0.53 # systemd-resolved's stub, named as NOT an answer 255.255.255.255 # limited broadcast, in the DHCP rebind note 169.254.169.254 # the cloud metadata address, named as a target diff --git a/test/drift.bats b/test/drift.bats index 9117397..ebf0384 100644 --- a/test/drift.bats +++ b/test/drift.bats @@ -313,3 +313,38 @@ stub_state() { [ "$(printf '%s\n' "$output" | grep -cx "$H")" -eq 0 ] } + +@test "an ephemeral port is not recorded as a listener" { + # `ss -l` with -u reports UDP sockets in UNCONN, which is what a client + # socket looks like: a resolver doing a lookup holds one on a random high + # port for as long as the query takes. So a box doing nothing produced a new + # PORT line every cycle and lost the one before it, which a watcher reads as + # a listener appearing and disappearing. + scratch="$BATS_TEST_TMPDIR/e1"; mkdir -p "$scratch" + drift_paths() { printf '%s\n' "$scratch"; } + DRIFT_ERRS="$BATS_TEST_TMPDIR/errs"; : > "$DRIFT_ERRS" + ss() { printf 'udp UNCONN 0 0 127.0.0.11:50267 0.0.0.0:*\nudp UNCONN 0 0 127.0.0.1:53 0.0.0.0:*\ntcp LISTEN 0 128 0.0.0.0:22 0.0.0.0:*\n'; } + export -f ss 2>/dev/null || true + run drift_state + [[ "$output" == *"PORT udp 127.0.0.1:53"* ]] + [[ "$output" == *"PORT tcp 0.0.0.0:22"* ]] + [[ "$output" != *50267* ]] +} + +@test "the same box twice running gives the same state" { + # The property that was lost. Two samples of a machine where nothing + # happened have to be identical, or every cycle is a finding. + scratch="$BATS_TEST_TMPDIR/e2"; mkdir -p "$scratch" + drift_paths() { printf '%s\n' "$scratch"; } + DRIFT_ERRS="$BATS_TEST_TMPDIR/errs"; : > "$DRIFT_ERRS" + ss() { printf 'udp UNCONN 0 0 127.0.0.11:%s 0.0.0.0:*\nudp UNCONN 0 0 127.0.0.1:53 0.0.0.0:*\n' "$((RANDOM % 20000 + 40000))"; } + export -f ss 2>/dev/null || true + a=$(drift_state); b=$(drift_state) + [ "$a" = "$b" ] +} + +@test "the ephemeral range comes from the kernel, not a constant" { + # It is tunable, and a wrong constant would either keep the flapping or hide + # real services. + grep -q 'ip_local_port_range' "$REPO/bin/dejima-devbox-audit" +} diff --git a/test/guard-shell.bats b/test/guard-shell.bats index fb1542e..8bac661 100644 --- a/test/guard-shell.bats +++ b/test/guard-shell.bats @@ -172,8 +172,20 @@ run_shell() { SSH_ORIGINAL_COMMAND="$1" AUDIT="$STUB/audit" PATH="$STUB:$PATH" r # one from a second deployment with its own baseline. An operator following # the note gets command-not-found on the step meant to confirm the upgrade. for f in "$REPO"/upgrades/0*.md; do - grep -qE '^dejima-guard (accept|check|trust-host)' "$f" || continue + grep -q 'dejima-guard \(accept\|check\|trust-host\)' "$f" || continue grep -q 'docker compose exec dejima-guard' "$f" \ || { echo "$f runs the guard without naming the container form"; false; } + + # BOTH forms, with the bare one first. dejima-guard does not always run + # in Docker, and a note giving only the prefixed command tells the + # operator of a host install to type something that will not work. + # v7 shipped that way, two days after this repository wrote the rule + # down in .dejima.md. + grep -qE '^dejima-guard (accept|check|trust-host)' "$f" \ + || { echo "$f gives only the container form"; false; } + bare=$(grep -nE '^dejima-guard (accept|check|trust-host)' "$f" | head -1 | cut -d: -f1) + pref=$(grep -n 'docker compose exec dejima-guard' "$f" | head -1 | cut -d: -f1) + [ "$bare" -lt "$pref" ] \ + || { echo "$f puts the container form before the plain one"; false; } done } diff --git a/upgrades/0007-ephemeral-ports.md b/upgrades/0007-ephemeral-ports.md new file mode 100644 index 0000000..38fd649 --- /dev/null +++ b/upgrades/0007-ephemeral-ports.md @@ -0,0 +1,78 @@ +--- +version: 7 +title: Stop reporting ephemeral sockets as listeners +layers: guest +snapshot: none +downtime: none +--- + +# v7: stop reporting ephemeral sockets as listeners + +Guest only, live-safe. **Fixes an alert that fires every cycle on a box where +nothing is happening**, introduced by v6. + +## Why + +v6 added UDP to the listening-socket collection, which was a real gap: a beacon +binding a datagram socket produced no line at all. + +`ss -l` with `-u` also reports UDP sockets in `UNCONN`, and that is what a +client socket looks like. A resolver doing a lookup holds one on a random high +port for as long as the query takes, and TCP has the same shape for anything +that binds before connecting. So a box doing nothing produced a new `PORT` line +every cycle and lost the one from the cycle before. + +To a watcher that is a listener appearing and disappearing. On dejima-guard it +lands as `new listener or unit` at max priority, roughly every half hour, and +`accept` does not help because the next sample differs again. The heartbeat +flaps with it: it fires only on a clean cycle, so the alerting ones send nothing +and the dead-man's switch reports the guard down and up in turn. + +Nothing was wrong with the watched machine on any of those cycles. + +## What changed + +Ports inside the kernel's ephemeral range are not recorded. The range is read +from `/proc/sys/net/ipv4/ip_local_port_range` rather than written down, since it +is tunable and a wrong constant would either keep the flapping or hide real +services. + +**What this gives up:** a listener the kernel would also hand out as an +ephemeral port is no longer recorded. That is a real loss and it is the better +trade, because the alternative is an alert every cycle, and an alert that fires +constantly is one nobody reads. + +## 1. Guest + +```sh +cd ~/dejima-devbox-hardener +git pull +sudo ./bin/dejima-devbox-bootstrap +``` + +Live-safe, and bootstrap records the version itself. + +## 2. Re-accept the baseline + +The ephemeral lines are gone from the state, so a baseline taken under v6 +differs from every state taken after this. + +On the machine running dejima-guard: + +```sh +dejima-guard accept +dejima-guard check # should be clean +``` + +**Running it in Docker?** There is no `dejima-guard` on that host, so every +command above takes a prefix: + +```sh +docker compose exec dejima-guard dejima-guard accept +``` + +Its own README has the rest, including the form to use when the container is +not running. + +If you do not run a guard at all, `sudo dejima-devbox-audit drift --accept` does +the same thing on the box itself.