From 40276fdb0aeb858ea47a1bf7ce3665b4e0fcca60 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Thu, 3 Sep 2026 11:31:36 -0700 Subject: [PATCH 1/2] Stop reporting ephemeral sockets as listeners Reported from a live host: "new listener or unit" at max priority every half hour, the heartbeat flapping down and up beside it, and accept not helping. No changes had been made to the watched machine. v6 added UDP to the listening-socket collection, which closed a real gap. But `ss -l` with -u also reports UDP sockets in UNCONN, and that is what a client socket looks like: a resolver doing a lookup holds one on a random high port for as long as the query takes, and TCP has the same shape for anything that binds before connecting. So a box doing nothing produced a new PORT line every cycle and lost the one from the cycle before. To a watcher that is a listener appearing and disappearing. accept cannot fix it because the next sample differs again, and the heartbeat only fires on a clean cycle, so the alerting ones send nothing and the dead-man's switch reports the guard down and up in turn. Every symptom on that host from one line of awk. Ports inside the kernel's ephemeral range are no longer recorded, with the range read from /proc rather than written down: it is tunable, and a wrong constant would either keep the flapping or hide real services. What this gives up is a listener the kernel would also hand out as an ephemeral port. That is a real loss and it is the better trade, because the alternative is an alert every cycle, and an alert that fires constantly is one nobody reads. v7, since the state no longer carries those lines and a v6 baseline differs from every state after this. The test that would have caught it asks the plain question the collector has to satisfy: two samples of a machine where nothing happened must be identical. --- VERSION | 2 +- bin/dejima-devbox-audit | 30 +++++++++++++- test/addresses.bats | 2 + test/drift.bats | 35 ++++++++++++++++ upgrades/0007-ephemeral-ports.md | 68 ++++++++++++++++++++++++++++++++ 5 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 upgrades/0007-ephemeral-ports.md diff --git a/VERSION b/VERSION index 1e8b314..7f8f011 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6 +7 diff --git a/bin/dejima-devbox-audit b/bin/dejima-devbox-audit index 8d32884..82dfe7a 100755 --- a/bin/dejima-devbox-audit +++ b/bin/dejima-devbox-audit @@ -679,7 +679,35 @@ drift_state() { # send queue. A flag added without moving the field would have recorded a # queue depth as a port, quietly, on every line. if command -v ss >/dev/null 2>&1; then - ss -H -ltun 2>/dev/null | awk 'NF {print "PORT " $1 " " $5}' + # EPHEMERAL PORTS ARE NOT LISTENERS, and including them made this state + # differ from itself every few minutes. + # + # `ss -l` with -u reports UDP sockets in UNCONN, which is what a client + # socket looks like: a resolver doing a lookup holds one on a random + # high port for as long as the query takes. TCP has the same shape for + # anything that binds before connecting. So a box doing nothing produced + # a new PORT line every cycle and lost the one from the cycle before, + # which a watcher reads as a listener appearing and disappearing - a max + # priority alert every half hour on a machine where nothing happened, + # and a baseline that cannot be accepted because the next sample differs + # again. + # + # The range comes from the kernel rather than being written down, since + # it is tunable and a wrong constant here would either keep the flapping + # or hide real services. + # + # What this gives up: a listener the kernel would also hand out as an + # ephemeral port is no longer recorded. That is a real loss, and it is + # the better trade - the alternative is an alert every cycle, and an + # alert that fires constantly is one nobody reads. + local ep_lo ep_hi + read -r ep_lo ep_hi < /proc/sys/net/ipv4/ip_local_port_range 2>/dev/null || true + : "${ep_lo:=32768}" "${ep_hi:=60999}" + ss -H -ltun 2>/dev/null | awk -v lo="$ep_lo" -v hi="$ep_hi" ' + NF { + n = split($5, a, ":"); port = a[n] + 0 + if (port < lo || port > hi) print "PORT " $1 " " $5 + }' fi return 0 } diff --git a/test/addresses.bats b/test/addresses.bats index a5528b2..3b7db39 100644 --- a/test/addresses.bats +++ b/test/addresses.bats @@ -43,6 +43,8 @@ setup() { allowed=( 0.0.0.0 # wildcard bind, sshd's ListenAddress 127.0.0.1 # loopback, as a rejected upstream resolver + 127.0.0.11 # Docker's embedded resolver, in the drift test that + # shows an ephemeral client socket is not a listener 127.0.0.53 # systemd-resolved's stub, named as NOT an answer 255.255.255.255 # limited broadcast, in the DHCP rebind note 169.254.169.254 # the cloud metadata address, named as a target diff --git a/test/drift.bats b/test/drift.bats index 9117397..ebf0384 100644 --- a/test/drift.bats +++ b/test/drift.bats @@ -313,3 +313,38 @@ stub_state() { [ "$(printf '%s\n' "$output" | grep -cx "$H")" -eq 0 ] } + +@test "an ephemeral port is not recorded as a listener" { + # `ss -l` with -u reports UDP sockets in UNCONN, which is what a client + # socket looks like: a resolver doing a lookup holds one on a random high + # port for as long as the query takes. So a box doing nothing produced a new + # PORT line every cycle and lost the one before it, which a watcher reads as + # a listener appearing and disappearing. + scratch="$BATS_TEST_TMPDIR/e1"; mkdir -p "$scratch" + drift_paths() { printf '%s\n' "$scratch"; } + DRIFT_ERRS="$BATS_TEST_TMPDIR/errs"; : > "$DRIFT_ERRS" + ss() { printf 'udp UNCONN 0 0 127.0.0.11:50267 0.0.0.0:*\nudp UNCONN 0 0 127.0.0.1:53 0.0.0.0:*\ntcp LISTEN 0 128 0.0.0.0:22 0.0.0.0:*\n'; } + export -f ss 2>/dev/null || true + run drift_state + [[ "$output" == *"PORT udp 127.0.0.1:53"* ]] + [[ "$output" == *"PORT tcp 0.0.0.0:22"* ]] + [[ "$output" != *50267* ]] +} + +@test "the same box twice running gives the same state" { + # The property that was lost. Two samples of a machine where nothing + # happened have to be identical, or every cycle is a finding. + scratch="$BATS_TEST_TMPDIR/e2"; mkdir -p "$scratch" + drift_paths() { printf '%s\n' "$scratch"; } + DRIFT_ERRS="$BATS_TEST_TMPDIR/errs"; : > "$DRIFT_ERRS" + ss() { printf 'udp UNCONN 0 0 127.0.0.11:%s 0.0.0.0:*\nudp UNCONN 0 0 127.0.0.1:53 0.0.0.0:*\n' "$((RANDOM % 20000 + 40000))"; } + export -f ss 2>/dev/null || true + a=$(drift_state); b=$(drift_state) + [ "$a" = "$b" ] +} + +@test "the ephemeral range comes from the kernel, not a constant" { + # It is tunable, and a wrong constant would either keep the flapping or hide + # real services. + grep -q 'ip_local_port_range' "$REPO/bin/dejima-devbox-audit" +} diff --git a/upgrades/0007-ephemeral-ports.md b/upgrades/0007-ephemeral-ports.md new file mode 100644 index 0000000..53b754d --- /dev/null +++ b/upgrades/0007-ephemeral-ports.md @@ -0,0 +1,68 @@ +--- +version: 7 +title: Stop reporting ephemeral sockets as listeners +layers: guest +snapshot: none +downtime: none +--- + +# v7: stop reporting ephemeral sockets as listeners + +Guest only, live-safe. **Fixes an alert that fires every cycle on a box where +nothing is happening**, introduced by v6. + +## Why + +v6 added UDP to the listening-socket collection, which was a real gap: a beacon +binding a datagram socket produced no line at all. + +`ss -l` with `-u` also reports UDP sockets in `UNCONN`, and that is what a +client socket looks like. A resolver doing a lookup holds one on a random high +port for as long as the query takes, and TCP has the same shape for anything +that binds before connecting. So a box doing nothing produced a new `PORT` line +every cycle and lost the one from the cycle before. + +To a watcher that is a listener appearing and disappearing. On dejima-guard it +lands as `new listener or unit` at max priority, roughly every half hour, and +`accept` does not help because the next sample differs again. The heartbeat +flaps with it: it fires only on a clean cycle, so the alerting ones send nothing +and the dead-man's switch reports the guard down and up in turn. + +Nothing was wrong with the watched machine on any of those cycles. + +## What changed + +Ports inside the kernel's ephemeral range are not recorded. The range is read +from `/proc/sys/net/ipv4/ip_local_port_range` rather than written down, since it +is tunable and a wrong constant would either keep the flapping or hide real +services. + +**What this gives up:** a listener the kernel would also hand out as an +ephemeral port is no longer recorded. That is a real loss and it is the better +trade, because the alternative is an alert every cycle, and an alert that fires +constantly is one nobody reads. + +## 1. Guest + +```sh +cd ~/dejima-devbox-hardener +git pull +sudo ./bin/dejima-devbox-bootstrap +``` + +Live-safe, and bootstrap records the version itself. + +## 2. Re-accept the baseline + +The ephemeral lines are gone from the state, so a baseline taken under v6 +differs from every state taken after this. + +On the machine running dejima-guard: + +```sh +docker compose exec dejima-guard dejima-guard accept +docker compose exec dejima-guard dejima-guard check +``` + +Without a container, drop the prefix. If you do not run a guard, +`sudo dejima-devbox-audit drift --accept` does the same on the box itself. From ac2fcc5487060496e69f4ce6f7a5891e09e3e5f3 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Thu, 3 Sep 2026 11:37:51 -0700 Subject: [PATCH 2/2] Give both forms of the guard command, with the plain one first dejima-guard does not always run in Docker, and v7's note gave only the prefixed form, mentioning the other as a trailing aside. An operator with a host install was told to type something that will not work there. This repository wrote the rule down in .dejima.md two days ago, and the test that was supposed to hold it only asked whether the container form was present - which v7 satisfied by having nothing else. It now asks for both, and for the plain one first, which is the order v5 and v6 use. --- test/guard-shell.bats | 14 +++++++++++++- upgrades/0007-ephemeral-ports.md | 18 ++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/test/guard-shell.bats b/test/guard-shell.bats index fb1542e..8bac661 100644 --- a/test/guard-shell.bats +++ b/test/guard-shell.bats @@ -172,8 +172,20 @@ run_shell() { SSH_ORIGINAL_COMMAND="$1" AUDIT="$STUB/audit" PATH="$STUB:$PATH" r # one from a second deployment with its own baseline. An operator following # the note gets command-not-found on the step meant to confirm the upgrade. for f in "$REPO"/upgrades/0*.md; do - grep -qE '^dejima-guard (accept|check|trust-host)' "$f" || continue + grep -q 'dejima-guard \(accept\|check\|trust-host\)' "$f" || continue grep -q 'docker compose exec dejima-guard' "$f" \ || { echo "$f runs the guard without naming the container form"; false; } + + # BOTH forms, with the bare one first. dejima-guard does not always run + # in Docker, and a note giving only the prefixed command tells the + # operator of a host install to type something that will not work. + # v7 shipped that way, two days after this repository wrote the rule + # down in .dejima.md. + grep -qE '^dejima-guard (accept|check|trust-host)' "$f" \ + || { echo "$f gives only the container form"; false; } + bare=$(grep -nE '^dejima-guard (accept|check|trust-host)' "$f" | head -1 | cut -d: -f1) + pref=$(grep -n 'docker compose exec dejima-guard' "$f" | head -1 | cut -d: -f1) + [ "$bare" -lt "$pref" ] \ + || { echo "$f puts the container form before the plain one"; false; } done } diff --git a/upgrades/0007-ephemeral-ports.md b/upgrades/0007-ephemeral-ports.md index 53b754d..38fd649 100644 --- a/upgrades/0007-ephemeral-ports.md +++ b/upgrades/0007-ephemeral-ports.md @@ -60,9 +60,19 @@ differs from every state taken after this. On the machine running dejima-guard: ```sh -docker compose exec dejima-guard dejima-guard accept -docker compose exec dejima-guard dejima-guard check +dejima-guard accept +dejima-guard check # should be clean ``` -Without a container, drop the prefix. If you do not run a guard, -`sudo dejima-devbox-audit drift --accept` does the same on the box itself. +**Running it in Docker?** There is no `dejima-guard` on that host, so every +command above takes a prefix: + +```sh +docker compose exec dejima-guard dejima-guard accept +``` + +Its own README has the rest, including the form to use when the container is +not running. + +If you do not run a guard at all, `sudo dejima-devbox-audit drift --accept` does +the same thing on the box itself.