Merge pull request 'Stop reporting ephemeral sockets as listeners' (#28) from ephemeral-ports into main

Reviewed-on: #28
Reviewed-by: branderson <brad@branderson.io>

VERSION

@@ -1 +1 @@
-6
+7

bin/dejima-devbox-audit

@@ -679,7 +679,35 @@ drift_state() {
     # send queue. A flag added without moving the field would have recorded a
     # queue depth as a port, quietly, on every line.
     if command -v ss >/dev/null 2>&1; then
-        ss -H -ltun 2>/dev/null | awk 'NF {print "PORT " $1 " " $5}'
+        # EPHEMERAL PORTS ARE NOT LISTENERS, and including them made this state
+        # differ from itself every few minutes.
+        #
+        # `ss -l` with -u reports UDP sockets in UNCONN, which is what a client
+        # socket looks like: a resolver doing a lookup holds one on a random
+        # high port for as long as the query takes. TCP has the same shape for
+        # anything that binds before connecting. So a box doing nothing produced
+        # a new PORT line every cycle and lost the one from the cycle before,
+        # which a watcher reads as a listener appearing and disappearing - a max
+        # priority alert every half hour on a machine where nothing happened,
+        # and a baseline that cannot be accepted because the next sample differs
+        # again.
+        #
+        # The range comes from the kernel rather than being written down, since
+        # it is tunable and a wrong constant here would either keep the flapping
+        # or hide real services.
+        #
+        # What this gives up: a listener the kernel would also hand out as an
+        # ephemeral port is no longer recorded. That is a real loss, and it is
+        # the better trade - the alternative is an alert every cycle, and an
+        # alert that fires constantly is one nobody reads.
+        local ep_lo ep_hi
+        read -r ep_lo ep_hi < /proc/sys/net/ipv4/ip_local_port_range 2>/dev/null || true
+        : "${ep_lo:=32768}" "${ep_hi:=60999}"
+        ss -H -ltun 2>/dev/null | awk -v lo="$ep_lo" -v hi="$ep_hi" '
+            NF {
+                n = split($5, a, ":"); port = a[n] + 0
+                if (port < lo || port > hi) print "PORT " $1 " " $5
+            }'
     fi
     return 0
 }

test/addresses.bats

@@ -43,6 +43,8 @@ setup() {
     allowed=(
         0.0.0.0             # wildcard bind, sshd's ListenAddress
         127.0.0.1           # loopback, as a rejected upstream resolver
+        127.0.0.11          # Docker's embedded resolver, in the drift test that
+                            # shows an ephemeral client socket is not a listener
         127.0.0.53          # systemd-resolved's stub, named as NOT an answer
         255.255.255.255     # limited broadcast, in the DHCP rebind note
         169.254.169.254     # the cloud metadata address, named as a target

test/drift.bats

@@ -313,3 +313,38 @@ stub_state() {
     [ "$(printf '%s\n' "$output" | grep -cx "$H")" -eq 0 ]
 }
 
+
+@test "an ephemeral port is not recorded as a listener" {
+    # `ss -l` with -u reports UDP sockets in UNCONN, which is what a client
+    # socket looks like: a resolver doing a lookup holds one on a random high
+    # port for as long as the query takes. So a box doing nothing produced a new
+    # PORT line every cycle and lost the one before it, which a watcher reads as
+    # a listener appearing and disappearing.
+    scratch="$BATS_TEST_TMPDIR/e1"; mkdir -p "$scratch"
+    drift_paths() { printf '%s\n' "$scratch"; }
+    DRIFT_ERRS="$BATS_TEST_TMPDIR/errs"; : > "$DRIFT_ERRS"
+    ss() { printf 'udp UNCONN 0 0 127.0.0.11:50267 0.0.0.0:*\nudp UNCONN 0 0 127.0.0.1:53 0.0.0.0:*\ntcp LISTEN 0 128 0.0.0.0:22 0.0.0.0:*\n'; }
+    export -f ss 2>/dev/null || true
+    run drift_state
+    [[ "$output" == *"PORT udp 127.0.0.1:53"* ]]
+    [[ "$output" == *"PORT tcp 0.0.0.0:22"* ]]
+    [[ "$output" != *50267* ]]
+}
+
+@test "the same box twice running gives the same state" {
+    # The property that was lost. Two samples of a machine where nothing
+    # happened have to be identical, or every cycle is a finding.
+    scratch="$BATS_TEST_TMPDIR/e2"; mkdir -p "$scratch"
+    drift_paths() { printf '%s\n' "$scratch"; }
+    DRIFT_ERRS="$BATS_TEST_TMPDIR/errs"; : > "$DRIFT_ERRS"
+    ss() { printf 'udp UNCONN 0 0 127.0.0.11:%s 0.0.0.0:*\nudp UNCONN 0 0 127.0.0.1:53 0.0.0.0:*\n' "$((RANDOM % 20000 + 40000))"; }
+    export -f ss 2>/dev/null || true
+    a=$(drift_state); b=$(drift_state)
+    [ "$a" = "$b" ]
+}
+
+@test "the ephemeral range comes from the kernel, not a constant" {
+    # It is tunable, and a wrong constant would either keep the flapping or hide
+    # real services.
+    grep -q 'ip_local_port_range' "$REPO/bin/dejima-devbox-audit"
+}

test/guard-shell.bats

@@ -172,8 +172,20 @@ run_shell() { SSH_ORIGINAL_COMMAND="$1" AUDIT="$STUB/audit" PATH="$STUB:$PATH" r
     # one from a second deployment with its own baseline. An operator following
     # the note gets command-not-found on the step meant to confirm the upgrade.
     for f in "$REPO"/upgrades/0*.md; do
-        grep -qE '^dejima-guard (accept|check|trust-host)' "$f" || continue
+        grep -q 'dejima-guard \(accept\|check\|trust-host\)' "$f" || continue
         grep -q 'docker compose exec dejima-guard' "$f" \
             || { echo "$f runs the guard without naming the container form"; false; }
+
+        # BOTH forms, with the bare one first. dejima-guard does not always run
+        # in Docker, and a note giving only the prefixed command tells the
+        # operator of a host install to type something that will not work.
+        # v7 shipped that way, two days after this repository wrote the rule
+        # down in .dejima.md.
+        grep -qE '^dejima-guard (accept|check|trust-host)' "$f" \
+            || { echo "$f gives only the container form"; false; }
+        bare=$(grep -nE '^dejima-guard (accept|check|trust-host)' "$f" | head -1 | cut -d: -f1)
+        pref=$(grep -n 'docker compose exec dejima-guard' "$f" | head -1 | cut -d: -f1)
+        [ "$bare" -lt "$pref" ] \
+            || { echo "$f puts the container form before the plain one"; false; }
     done
 }

upgrades/0007-ephemeral-ports.md

@@ -0,0 +1,78 @@
+---
+version: 7
+title: Stop reporting ephemeral sockets as listeners
+layers: guest
+snapshot: none
+downtime: none
+---
+
+# v7: stop reporting ephemeral sockets as listeners
+
+Guest only, live-safe. **Fixes an alert that fires every cycle on a box where
+nothing is happening**, introduced by v6.
+
+## Why
+
+v6 added UDP to the listening-socket collection, which was a real gap: a beacon
+binding a datagram socket produced no line at all.
+
+`ss -l` with `-u` also reports UDP sockets in `UNCONN`, and that is what a
+client socket looks like. A resolver doing a lookup holds one on a random high
+port for as long as the query takes, and TCP has the same shape for anything
+that binds before connecting. So a box doing nothing produced a new `PORT` line
+every cycle and lost the one from the cycle before.
+
+To a watcher that is a listener appearing and disappearing. On dejima-guard it
+lands as `new listener or unit` at max priority, roughly every half hour, and
+`accept` does not help because the next sample differs again. The heartbeat
+flaps with it: it fires only on a clean cycle, so the alerting ones send nothing
+and the dead-man's switch reports the guard down and up in turn.
+
+Nothing was wrong with the watched machine on any of those cycles.
+
+## What changed
+
+Ports inside the kernel's ephemeral range are not recorded. The range is read
+from `/proc/sys/net/ipv4/ip_local_port_range` rather than written down, since it
+is tunable and a wrong constant would either keep the flapping or hide real
+services.
+
+**What this gives up:** a listener the kernel would also hand out as an
+ephemeral port is no longer recorded. That is a real loss and it is the better
+trade, because the alternative is an alert every cycle, and an alert that fires
+constantly is one nobody reads.
+
+## 1. Guest
+
+```sh
+cd ~/dejima-devbox-hardener
+git pull
+sudo ./bin/dejima-devbox-bootstrap
+```
+
+Live-safe, and bootstrap records the version itself.
+
+## 2. Re-accept the baseline
+
+The ephemeral lines are gone from the state, so a baseline taken under v6
+differs from every state taken after this.
+
+On the machine running dejima-guard:
+
+```sh
+dejima-guard accept
+dejima-guard check      # should be clean
+```
+
+**Running it in Docker?** There is no `dejima-guard` on that host, so every
+command above takes a prefix:
+
+```sh
+docker compose exec dejima-guard dejima-guard accept
+```
+
+Its own README has the rest, including the form to use when the container is
+not running.
+
+If you do not run a guard at all, `sudo dejima-devbox-audit drift --accept` does
+the same thing on the box itself.