Merge branch 'main' into ttyd-mobile-client

docker/dns-proxy.sh

@@ -10,7 +10,11 @@
 # resolver; everything else gets no upstream configured at all, so it fails
 # outright instead of leaking the query. Must run before init-firewall.sh so
 # that script's own dig calls (and everything afterward) go through it.
-set -euo pipefail
+set -Eeuo pipefail
+# Same reasoning as init-firewall.sh's: this runs before the firewall in the
+# container's startup, and a silent `set -e` abort here is indistinguishable
+# from the sandbox never starting for any other reason (#59).
+trap 'status=$?; echo "ERROR: dns-proxy.sh aborted at line $LINENO (exit $status): $BASH_COMMAND" >&2' ERR
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=fixed-domains.sh

docker/entrypoint.sh

@@ -1,5 +1,12 @@
 #!/bin/bash
-set -euo pipefail
+set -Eeuo pipefail
+# The container's startup has to be able to say why it stopped. Under `set -e`
+# alone it cannot: the script aborts with no output, the container exits, and
+# the only symptom anywhere is that the session never appeared - which reads as
+# the sandbox being broken rather than as one command having failed. This is
+# also what makes such a failure distinguishable from the checks below that
+# refuse deliberately, and say so (#59).
+trap 'status=$?; echo "ERROR: entrypoint.sh aborted at line $LINENO (exit $status): $BASH_COMMAND" >&2' ERR
 
 # Derive the Gitea API hostname from GITEA_API (a full URL) so it can be
 # firewall-allowlisted alongside GITEA_SSH_HOST.

docker/init-firewall.sh

@@ -8,9 +8,28 @@
 # everything here - IPv6 blocking, ulogd2 verdict logging, the GitHub
 # push/pull split, GH_TOKEN auth, and the extra-domains allowlist - is this
 # project's own.
-set -euo pipefail
+set -Eeuo pipefail
 IFS=$'\n\t'
 
+# Nothing in here may die quietly. Under `set -e` a failing command aborts the
+# script with no output at all, and this script's failures are invisible in the
+# worst possible way: the container simply never appears, which reads as the
+# sandbox being broken rather than as one command in one loop having failed.
+# Three CI failures were spent on two different instances of exactly that (#59)
+# - one at the GitHub ranges fetch, one in the domain loop below - and neither
+# left a single line to go on.
+#
+# The explicit `exit 1` paths below say more than this can and are worth
+# keeping; this is the floor under everything else. -E on the line above is
+# what extends it into functions and subshells, where it would otherwise not
+# fire at all.
+#
+# A failure inside a command substitution reports twice - once from the
+# subshell, naming the command that actually failed, and once from the
+# statement containing it. That is worth the noise: for `x=$(a | b)` the first
+# line is the only thing that says whether it was `a` or `b`.
+trap 'status=$?; echo "ERROR: init-firewall.sh aborted at line $LINENO (exit $status): $BASH_COMMAND" >&2' ERR
+
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=fixed-domains.sh
 source "$SCRIPT_DIR/fixed-domains.sh"
@@ -86,22 +105,61 @@ gh_auth=()
 if [ -n "${GH_TOKEN:-}" ]; then
     gh_auth=(-H "Authorization: Bearer $GH_TOKEN")
 fi
-gh_ranges=$(curl -s ${gh_auth[@]+"${gh_auth[@]}"} https://api.github.com/meta)
-if [ -z "$gh_ranges" ]; then
-    echo "ERROR: Empty response fetching GitHub's IP ranges"
-    exit 1
-fi
 
-if ! echo "$gh_ranges" | jq -e '.web and .api and .git' >/dev/null; then
-    # An API error (rate limiting, a bad token) returns a JSON body with a
-    # `message` and no ranges - non-empty, so it reaches here rather than the
-    # emptiness check above. Surface it instead of reporting the response as
-    # malformed, which points at the wrong thing entirely.
-    if gh_error=$(echo "$gh_ranges" | jq -re '.message' 2>/dev/null); then
-        echo "ERROR: GitHub API request failed: $gh_error"
-    else
-        echo "ERROR: GitHub API response missing required fields"
-    fi
+# Bounded, retried, and reported. All three were missing, and the combination
+# was worse than any of them alone (#59).
+#
+# `curl -s` with no --max-time has no deadline at all - it waits as long as the
+# kernel keeps retrying the SYN. And under `set -euo pipefail` at the top of
+# this file, a curl that exits non-zero takes the whole script down ON THE
+# ASSIGNMENT, so every diagnostic below it is unreachable in exactly the case
+# that needs one: the container dies having printed nothing after the line
+# above, and the operator - or CI - sees only "the container never became
+# ready". The `|| status=$?` is what keeps the failure ours to report.
+#
+# Retried because both failure modes here are transient. A blip costs a few
+# seconds instead of the launch; and a burst of container starts can trip
+# GitHub's secondary rate limit, which answers 403 with a JSON body rather
+# than a transport error - so the retry has to wrap the response check too,
+# not just the transfer. curl's own --retry would not cover that case.
+fetch_github_ranges() {
+    local attempt status body
+    for attempt in 1 2 3; do
+        status=0
+        # -S so curl's own reason ("Operation timed out after ...") reaches
+        # stderr and therefore `docker logs`; -s alone would swallow it.
+        body=$(curl -sS --connect-timeout 5 --max-time 10 \
+            ${gh_auth[@]+"${gh_auth[@]}"} https://api.github.com/meta) || status=$?
+        if [ "$status" -ne 0 ]; then
+            echo "  attempt $attempt/3: curl exited $status" >&2
+        elif [ -z "$body" ]; then
+            echo "  attempt $attempt/3: empty response" >&2
+        elif ! printf '%s' "$body" | jq -e '.web and .api and .git' >/dev/null 2>&1; then
+            # An API error - rate limiting, a bad token - comes back as a JSON
+            # body carrying a `message` and no ranges. Surface it rather than
+            # reporting the response as malformed, which points at the wrong
+            # thing entirely.
+            echo "  attempt $attempt/3: $(printf '%s' "$body" \
+                | jq -re '.message' 2>/dev/null || echo 'response carried no ranges')" >&2
+        else
+            printf '%s' "$body"
+            return 0
+        fi
+        # `if`, not `[ ... ] && sleep`: the AND-list's status on the last
+        # attempt is 1, which set -e would take as the script failing here
+        # rather than falling through to the message below.
+        if [ "$attempt" -lt 3 ]; then
+            sleep $((attempt * 2))
+        fi
+    done
+    return 1
+}
+
+if ! gh_ranges=$(fetch_github_ranges); then
+    echo "ERROR: could not fetch GitHub's IP ranges from api.github.com/meta." >&2
+    echo "Tried three times over ~36s; the reason for each attempt is above." >&2
+    echo "The sandbox cannot start without them - the firewall would deny every" >&2
+    echo "GitHub range, so git, gh and the API would all fail once inside." >&2
     exit 1
 fi
 
@@ -122,11 +180,48 @@ done < <(echo "$gh_ranges" | jq -r '(.web + .api + .git)[]' | aggregate -q)
 
 # Resolve and add other allowed domains (shared with dns-proxy.sh's name
 # allowlist via fixed-domains.sh, so the two can't drift apart)
+# Same treatment as the GitHub fetch above, and for the same reason: this loop
+# was `ips=$(dig ... | awk ...)`, which under `set -o pipefail` dies on the
+# ASSIGNMENT when dig exits non-zero - silently, before the "returned nothing"
+# check below can run. It aborted a CI container twice, at a different domain
+# each time, with the log simply stopping mid-loop (#59).
+#
+# Retried because a lookup here is the first traffic through a dnsmasq that
+# entrypoint.sh started moments ago, and because a CDN's resolver being briefly
+# unavailable should cost a second, not the container. +time/+tries bound each
+# attempt: dig's defaults are 5s x 3 tries, so a single unanswered lookup could
+# spend 15s of a startup budget on its own.
+resolve_domain() {
+    local domain="$1" attempt status answer ips
+    for attempt in 1 2 3; do
+        status=0
+        answer=$(dig +noall +answer +time=3 +tries=1 A "$domain" 2>&1) || status=$?
+        if [ "$status" -ne 0 ]; then
+            echo "  attempt $attempt/3: dig exited $status${answer:+ - $answer}" >&2
+        else
+            ips=$(printf '%s\n' "$answer" | awk '$4 == "A" {print $5}')
+            if [ -n "$ips" ]; then
+                printf '%s\n' "$ips"
+                return 0
+            fi
+            # A CNAME-only answer, an empty one, or a rebind rejection. Printing
+            # what did come back separates "no such name" from "dnsmasq refused
+            # this answer", which look identical from an empty A list.
+            echo "  attempt $attempt/3: no A records${answer:+ - $answer}" >&2
+        fi
+        if [ "$attempt" -lt 3 ]; then
+            sleep "$attempt"
+        fi
+    done
+    return 1
+}
+
 for domain in "${FIXED_DOMAINS[@]}"; do
     echo "Looking up $domain..."
-    ips=$(dig +noall +answer A "$domain" | awk '$4 == "A" {print $5}')
-    if [ -z "$ips" ]; then
-        echo "ERROR: DNS lookup for $domain returned nothing"
+    if ! ips=$(resolve_domain "$domain"); then
+        echo "ERROR: DNS lookup for $domain failed - each attempt is above." >&2
+        echo "The firewall cannot allowlist a domain it cannot resolve, and the" >&2
+        echo "sandbox would start with that domain unreachable, so this is fatal." >&2
         exit 1
     fi
 
@@ -164,8 +259,12 @@ if [ -n "${EXTRA_ALLOWED_DOMAINS:-}" ]; then
             continue
         fi
         echo "Resolving $domain (extra)..."
-        ips=$(dig +noall +answer A "$domain" | awk '$4 == "A" {print $5}')
-        if [ -z "$ips" ]; then
+        # Through resolve_domain like the fixed domains, which is what makes
+        # the comment above true. `ips=$(dig ... | awk ...)` did not merely
+        # fail to warn on a failing dig - under `set -o pipefail` it took the
+        # whole script down on the assignment, so "non-fatal" held for a
+        # domain that resolved to nothing and not for one whose lookup errored.
+        if ! ips=$(resolve_domain "$domain"); then
             echo "WARNING: Failed to resolve extra domain $domain, skipping"
             continue
         fi
@@ -241,7 +340,12 @@ iptables -A OUTPUT -j NFLOG --nflog-group 2 --nflog-prefix "reject"
 iptables -A OUTPUT -j REJECT --reject-with icmp-admin-prohibited
 
 echo "Firewall rules applied. Self-checking..."
-if curl --connect-timeout 5 https://example.com >/dev/null 2>&1; then
+# --max-time as well as --connect-timeout on both self-checks: a connect that
+# succeeds and then stalls is not covered by --connect-timeout, and while these
+# two sit inside `if` (so set -e cannot take the script down the way it could
+# above), a stalled transfer here still hangs the container's startup with no
+# deadline of its own.
+if curl --connect-timeout 5 --max-time 15 https://example.com >/dev/null 2>&1; then
     echo "ERROR: self-check failed - a non-allowlisted host (example.com) was reachable"
     exit 1
 else
@@ -249,7 +353,7 @@ else
 fi
 
 # Confirm the allowlist itself actually works, not just that the default-deny does.
-if ! curl --connect-timeout 5 https://api.github.com/zen >/dev/null 2>&1; then
+if ! curl --connect-timeout 5 --max-time 15 https://api.github.com/zen >/dev/null 2>&1; then
     echo "ERROR: self-check failed - api.github.com should be reachable but wasn't"
     exit 1
 else

test/runtime.bats

@@ -45,19 +45,91 @@ network_name() { echo "dejima-test-net-$(basename "$BATS_FILE_TMPDIR")"; }
 # assertions into vacuous passes, since everything is still reachable at
 # that point. Waiting for `sleep infinity` means the whole startup sequence
 # (DNS proxy, firewall, ipsets, logging, forwarder bind) has completed.
+#
+# 120 rather than 60: init-firewall.sh's GitHub ranges fetch now retries, and a
+# run that loses two attempts before succeeding spends ~26s of the budget on
+# that call alone (#59) - which on a runner that has just spent three minutes
+# unpacking an image is enough to turn a recovery into a spurious failure. A
+# longer budget only delays reporting a genuinely stuck startup; it cannot hide
+# one, since nothing here passes on a timeout.
+# `docker wait` returns the moment the container exits, but the json-file log
+# driver writes asynchronously - so reading the log right then can return one
+# that stops mid-stream, with everything after some flush boundary missing.
+#
+# That is not hypothetical. It presented as a container "aborting" partway
+# through init-firewall.sh's domain loop, at a slightly different domain each
+# run, and survived three separate fixes to that loop before the giveaway: the
+# log ended with no message from an ERR trap that cannot be bypassed and no
+# output from a retry that prints on every path. Nothing had aborted. The read
+# was short.
+#
+# Two identical consecutive reads means the driver has caught up. Deliberately
+# content-neutral - waiting for an expected string would make the wait pass or
+# fail on the very thing the caller is about to assert.
+settled_logs() {
+    local container="$1" current previous=""
+    for _ in $(seq 1 20); do
+        current=$(docker logs "$container" 2>&1)
+        if [ -n "$current" ] && [ "$current" = "$previous" ]; then
+            break
+        fi
+        previous="$current"
+        sleep 0.25
+    done
+    printf '%s' "$current"
+}
+
 wait_ready() {
     local container="$1"
-    for _ in $(seq 1 60); do
+    for _ in $(seq 1 120); do
         if docker exec "$container" pgrep -x -f 'sleep infinity' >/dev/null 2>&1; then
             return 0
         fi
         sleep 1
     done
     echo "container $container never reached the post-entrypoint sleep - startup log:" >&2
-    docker logs "$container" >&2
+    settled_logs "$container" >&2
     return 1
 }
 
+# $IMAGE is a single mutable tag on a shared daemon, and SKIP_BUILD=1 - which
+# CI sets, having built the image in an earlier step - means this file does not
+# build it either. Nothing in that arrangement ties the image to the checkout,
+# and when the two disagree every assertion in this file is being made against
+# code that is not under test.
+#
+# That is not a theoretical worry. It cost six CI rounds: a container kept
+# stopping partway through init-firewall.sh, and the fixes that were supposed
+# to make it explain itself - an ERR trap, a resolver that reports on every
+# path - produced no output at all, because the script running in the container
+# was not the script in the commit. A stale image looks exactly like a bug in
+# the code you are reading, which is the worst way for it to present.
+#
+# One container start, three files, compared by content rather than by any
+# claim about how the image was produced.
+assert_image_matches_checkout() {
+    local expected actual
+    expected=$(cd "$SANDBOX_DIR" && md5sum init-firewall.sh dns-proxy.sh entrypoint.sh | awk '{print $1}')
+    actual=$(docker run --rm --entrypoint sh "$IMAGE" -c \
+        'cd /usr/local/bin && md5sum init-firewall.sh dns-proxy.sh entrypoint.sh' 2>&1 | awk '{print $1}')
+    if [ "$expected" != "$actual" ]; then
+        echo "The image tagged $IMAGE was not built from this checkout." >&2
+        echo >&2
+        echo "checkout (docker/):        $(echo "$expected" | tr '\n' ' ')" >&2
+        echo "image (/usr/local/bin/):   $(echo "$actual" | tr '\n' ' ')" >&2
+        echo >&2
+        echo "Every test in this file would be asserting against different code than" >&2
+        echo "the one in the working tree, so they are not run. Rebuild it:" >&2
+        echo >&2
+        echo "  docker build -t $IMAGE -f docker/Dockerfile ." >&2
+        echo >&2
+        echo "In CI this means the build step and the test step disagreed - most" >&2
+        echo "likely another job retagged $IMAGE in between, since the tag is shared" >&2
+        echo "and carries no version." >&2
+        return 1
+    fi
+}
+
 setup_file() {
     SANDBOX_DIR="$BATS_TEST_DIRNAME/../docker"
     REPO_ROOT="$BATS_TEST_DIRNAME/.."
@@ -68,6 +140,7 @@ setup_file() {
     if [ "${SKIP_BUILD:-0}" != "1" ]; then
         docker build -q -t "$IMAGE" -f "$SANDBOX_DIR/Dockerfile" "$REPO_ROOT" >&2
     fi
+    assert_image_matches_checkout
 
     docker rm -f "$container" >/dev/null 2>&1 || true
     docker network rm "$network" >/dev/null 2>&1 || true
@@ -1050,26 +1123,66 @@ EOF
     # this container took its loopback one, and every API call in the session
     # would fail with nothing naming the cause.
     container="dejima-test-badtransport-$(basename "$BATS_FILE_TMPDIR")"
-    docker run -d --name "$container" \
+    docker rm -f "$container" >/dev/null 2>&1 || true
+
+    # Run in the FOREGROUND and read the container's own streams, rather than
+    # detaching and asking the daemon's log driver for them afterwards.
+    #
+    # The detached form spent six CI rounds insisting this container aborted
+    # partway through init-firewall.sh. It had not. Every line those logs ever
+    # contained came from an `echo` to stdout, and not one line written to
+    # stderr appeared in any of them - not init-firewall.sh's ERR trap, not
+    # resolve_domain's per-attempt reports, and not the FATAL message this test
+    # asserts on, which entrypoint.sh writes to stderr as well. The container
+    # was doing the right thing; the stderr half of what it said was not
+    # arriving.
+    #
+    # A foreground `docker run` attaches to the container's streams directly,
+    # so nothing is buffered, flushed or dropped by a log driver in between.
+    # It is also simply less machinery: one command yields both the output and
+    # the exit status, with no wait, no settling and no read-after-exit race.
+    #
+    # The timeout stays: without the transport check the entrypoint runs on
+    # into SANDBOX_TEST_MODE's `sleep infinity` and never exits, and that
+    # regression should fail here rather than hang the suite.
+    run timeout 120 docker run --name "$container" \
         --cap-add=NET_ADMIN --cap-add=NET_RAW \
         --network "$(network_name)" \
         -e SANDBOX_TEST_MODE=1 \
         -e GH_TOKEN="${GH_TOKEN:-}" \
         -e SANDBOX_ANTHROPIC_TRANSPORT=Loopback \
         --entrypoint /usr/local/bin/entrypoint.sh \
-        "$IMAGE" >/dev/null
+        "$IMAGE"
+    exit_code=$status
+    logs=$output
 
-    # Without the check the entrypoint runs on into SANDBOX_TEST_MODE's `sleep
-    # infinity` and never exits - the timeout turns that regression into a
-    # failure here rather than a hung suite.
-    run timeout 120 docker wait "$container"
-    exit_code=$output
-    logs=$(docker logs "$container" 2>&1)
+    # Still worth capturing: it answers what the output cannot - whether the
+    # script exited or the container was killed underneath it.
+    state=$(docker inspect \
+        --format 'exit={{.State.ExitCode}} oom={{.State.OOMKilled}} error={{printf "%q" .State.Error}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' \
+        "$container" 2>&1 || true)
 
     docker rm -f "$container" >/dev/null 2>&1 || true
 
-    [ "$exit_code" = "1" ]
-    [[ "$logs" == *"SANDBOX_ANTHROPIC_TRANSPORT is set to 'Loopback'"* ]]
-    [[ "$logs" == *"'tls'"* ]]
-    [[ "$logs" == *"'loopback'"* ]]
+    # Every assertion prints what it was reading. bats prints the failing
+    # expression, not the variable, and the container is gone by then - so
+    # without this the whole failure is "the container did not say what we
+    # expected", which is equally true when the container aborted early, when
+    # it was killed, and when its output never reached us. Those took six CI
+    # rounds to tell apart (#59); the byte count and the state are what
+    # separate them.
+    dump() {
+        echo "$1" >&2
+        echo "exit code: $exit_code" >&2
+        echo "container state: $state" >&2
+        echo "container output (${#logs} bytes):" >&2
+        echo "$logs" >&2
+        false
+    }
+
+    [ "$exit_code" = "1" ] || dump "expected the container to exit 1"
+    [[ "$logs" == *"SANDBOX_ANTHROPIC_TRANSPORT is set to 'Loopback'"* ]] \
+        || dump "the transport check never ran - something aborted the entrypoint before it"
+    [[ "$logs" == *"'tls'"* ]] || dump "the refusal did not name the valid values"
+    [[ "$logs" == *"'loopback'"* ]] || dump "the refusal did not name the valid values"
 }

test/static.bats

@@ -72,6 +72,298 @@ setup() {
     }
 }
 
+# The runtime counterpart of the test above. Build-time downloads have had a
+# retry guard for a while; the fetches that run on every container START had
+# nothing, which is the worse place for it - a dead build is obvious, and a
+# container that never comes up reads as the sandbox being broken.
+@test "every fetch in the container's startup path has a deadline" {
+    # curl has no default overall timeout. --connect-timeout alone is not one
+    # either: it bounds reaching the far end, not a transfer that connects and
+    # then stalls, which is the shape that hangs a startup indefinitely.
+    local unguarded
+    unguarded=$(awk '
+        /^[[:space:]]*#/ { next }
+        /curl/ && /https?:\/\// && !/--max-time [0-9]/ { print FILENAME": "FNR": "$0 }
+    ' "$SANDBOX_DIR"/*.sh)
+    [ -z "$unguarded" ] || {
+        echo "these runtime fetches have no --max-time, so they can hang a container start:" >&2
+        echo "$unguarded" >&2
+        false
+    }
+}
+
+@test "the GitHub ranges fetch reports its own failure instead of letting set -e swallow it" {
+    # init-firewall.sh runs under `set -euo pipefail`, so a bare
+    # `var=$(curl ...)` that fails takes the whole script down ON THE
+    # ASSIGNMENT - before any of the diagnostics written for exactly that case
+    # can run. The container then exits having printed nothing after "Looking
+    # up GitHub's IP ranges...", and the only symptom anyone sees is that the
+    # sandbox did not start. It cost two CI rounds to not diagnose (#59).
+    #
+    # The fix is to capture curl's status rather than let set -e have it, so
+    # this checks the fetch is not a bare assignment again.
+    grep -q 'set -euo pipefail' "$SANDBOX_DIR/init-firewall.sh"
+    # Continuations are joined before matching: the fetch spans two lines, and
+    # the `||` that makes it safe is on the second one - so a line-at-a-time
+    # pass would report the correct form as the broken one.
+    local bare
+    bare=$(awk '
+        /^[[:space:]]*#/ { next }
+        {
+            line = $0
+            start = FNR
+            while (line ~ /\\$/ && (getline) > 0) {
+                sub(/\\$/, "", line)
+                line = line $0
+            }
+            if (line ~ /^[[:space:]]*[A-Za-z_][A-Za-z0-9_]*=\$\(curl/ && line !~ /\|\|/)
+                print start": "line
+        }
+    ' "$SANDBOX_DIR/init-firewall.sh")
+    [ -z "$bare" ] || {
+        echo "this fetch is a bare assignment, so set -e will kill the script before it can report:" >&2
+        echo "$bare" >&2
+        false
+    }
+    # And that the failure path actually says something. The message is what
+    # separates "GitHub was unreachable" from "the sandbox is broken".
+    grep -q "could not fetch GitHub's IP ranges" "$SANDBOX_DIR/init-firewall.sh"
+}
+
+@test "every startup script reports where it aborted instead of dying silently" {
+    # `set -e` alone aborts with no output whatsoever. For a script that runs
+    # during container startup that is the worst possible failure mode: the
+    # container just never appears, and every cause - a failed lookup, a full
+    # ipset, an unreachable API - presents identically, as "the sandbox is
+    # broken". Three CI failures were spent on two different instances of it.
+    #
+    # log-shipper.sh is deliberately absent: it does not use -e at all, on
+    # purpose, because one bad line should not kill the tailer.
+    local script
+    for script in init-firewall.sh dns-proxy.sh entrypoint.sh; do
+        # -E, not just -e: without errtrace the handler does not fire inside
+        # functions or subshells, which is where the fetch and the loops live.
+        grep -qE '^set -Eeuo pipefail$' "$SANDBOX_DIR/$script" \
+            || { echo "$script does not set -E, so its ERR trap would not fire in functions" >&2; false; }
+        grep -q "trap .*ERR" "$SANDBOX_DIR/$script" \
+            || { echo "$script has no ERR trap - a set -e abort in it would be silent" >&2; false; }
+        # Naming itself matters: three scripts run back to back into the same
+        # log, and "aborted at line 128" is useless without knowing whose.
+        grep -q "aborted at line" "$SANDBOX_DIR/$script"
+        grep -q "$script aborted" "$SANDBOX_DIR/$script" \
+            || { echo "$script's ERR trap does not name $script" >&2; false; }
+    done
+}
+
+@test "the ERR trap actually resolves the line and the command" {
+    # Extracted from the real file rather than retyped, so a trap that has been
+    # broken into single quotes - which would expand $LINENO and $BASH_COMMAND
+    # at definition time and report the trap's own line forever - fails here.
+    local trapline
+    trapline=$(grep -m1 "trap .*ERR" "$SANDBOX_DIR/init-firewall.sh")
+    cat > "$BATS_TEST_TMPDIR/probe.sh" <<EOF
+set -Eeuo pipefail
+$trapline
+echo "Looking up files.pythonhosted.org..."
+false
+echo "should not be reached"
+EOF
+    run bash "$BATS_TEST_TMPDIR/probe.sh"
+    [ "$status" -ne 0 ]
+    [[ "$output" != *"should not be reached"* ]]
+    [[ "$output" == *"init-firewall.sh aborted at line 4"* ]] \
+        || { echo "trap did not report the failing line:" >&2; echo "$output" >&2; false; }
+    [[ "$output" == *"false"* ]] \
+        || { echo "trap did not report the failing command:" >&2; echo "$output" >&2; false; }
+}
+
+# The behaviour behind the two greps above, driven for real. init-firewall.sh
+# cannot be sourced - it flushes iptables at the top level - so the function is
+# extracted and run against a stubbed curl. No container, no network.
+#
+# `sleep` is shadowed by a no-op: the retry delays are deliberate in production
+# and pure cost here.
+run_fetch() {
+    local behaviour="$1"
+    STUB_DIR="$BATS_TEST_TMPDIR/stub"
+    mkdir -p "$STUB_DIR"
+    cp "$behaviour" "$STUB_DIR/curl"
+    chmod +x "$STUB_DIR/curl"
+    PATH="$STUB_DIR:$PATH"
+    sleep() { :; }
+    eval "$(sed -n '/^fetch_github_ranges()/,/^}/p' "$SANDBOX_DIR/init-firewall.sh")"
+    gh_auth=()
+    fetch_github_ranges
+}
+
+# A minimal but structurally valid /meta response - enough for the .web/.api/
+# .git check the function makes.
+meta_json() {
+    printf '{"web":["192.0.2.0/24"],"api":["192.0.2.0/24"],"git":["192.0.2.0/24"]}'
+}
+
+@test "the ranges fetch returns the body when GitHub answers" {
+    cat > "$BATS_TEST_TMPDIR/ok" <<EOF
+#!/bin/bash
+printf '%s' '$(meta_json)'
+EOF
+    run run_fetch "$BATS_TEST_TMPDIR/ok"
+    [ "$status" -eq 0 ]
+    [ "$output" = "$(meta_json)" ]
+}
+
+@test "the ranges fetch reports a failing curl rather than dying on the assignment" {
+    # 28 is curl's operation-timeout. Before the fix this case never reached a
+    # message at all: set -e took the script down on the assignment itself.
+    cat > "$BATS_TEST_TMPDIR/timeout" <<'EOF'
+#!/bin/bash
+echo "curl: (28) Operation timed out" >&2
+exit 28
+EOF
+    run run_fetch "$BATS_TEST_TMPDIR/timeout"
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"attempt 1/3: curl exited 28"* ]]
+    [[ "$output" == *"attempt 3/3: curl exited 28"* ]]
+    # curl's own reason has to survive too - it is the difference between
+    # "GitHub was slow" and "DNS is broken in here".
+    [[ "$output" == *"Operation timed out"* ]]
+}
+
+@test "the ranges fetch retries an API error body and surfaces its message" {
+    # A rate limit answers 200-with-a-message or 403-with-a-message, not a
+    # transport error, so curl exits 0 and only the response check catches it.
+    # A burst of container starts is exactly what trips it, and it clears on
+    # its own - so it has to be retried, which curl's own --retry would not do.
+    cat > "$BATS_TEST_TMPDIR/ratelimited" <<'EOF'
+#!/bin/bash
+printf '%s' '{"message":"API rate limit exceeded"}'
+EOF
+    run run_fetch "$BATS_TEST_TMPDIR/ratelimited"
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"API rate limit exceeded"* ]]
+    [[ "$output" == *"attempt 3/3"* ]]
+}
+
+@test "the ranges fetch recovers when a later attempt succeeds" {
+    # The point of retrying at all: one blip must cost seconds, not the launch.
+    cat > "$BATS_TEST_TMPDIR/flaky" <<EOF
+#!/bin/bash
+n="\$BATS_TEST_TMPDIR/attempts"
+count=\$(cat "\$n" 2>/dev/null || echo 0)
+echo \$((count + 1)) > "\$n"
+if [ "\$count" -lt 1 ]; then exit 7; fi
+printf '%s' '$(meta_json)'
+EOF
+    run run_fetch "$BATS_TEST_TMPDIR/flaky"
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"attempt 1/3: curl exited 7"* ]]
+    [[ "$output" == *"$(meta_json)"* ]]
+}
+
+# Same harness, for the domain loop - the other place a silent `set -e` abort
+# stopped a container mid-startup, at a different domain each time.
+run_resolve() {
+    local behaviour="$1" domain="${2:-files.pythonhosted.org}"
+    STUB_DIR="$BATS_TEST_TMPDIR/stub"
+    mkdir -p "$STUB_DIR"
+    cp "$behaviour" "$STUB_DIR/dig"
+    chmod +x "$STUB_DIR/dig"
+    PATH="$STUB_DIR:$PATH"
+    sleep() { :; }
+    eval "$(sed -n '/^resolve_domain()/,/^}/p' "$SANDBOX_DIR/init-firewall.sh")"
+    resolve_domain "$domain"
+}
+
+@test "domain resolution returns the A records dig answered with" {
+    cat > "$BATS_TEST_TMPDIR/ok" <<'EOF'
+#!/bin/bash
+echo "files.pythonhosted.org.	60	IN	A	151.101.64.223"
+echo "files.pythonhosted.org.	60	IN	A	151.101.128.223"
+EOF
+    run run_resolve "$BATS_TEST_TMPDIR/ok"
+    [ "$status" -eq 0 ]
+    [ "$output" = "151.101.64.223
+151.101.128.223" ]
+}
+
+@test "a failing dig is reported rather than killing the script on the assignment" {
+    # This is the one that aborted CI: `ips=$(dig ... | awk ...)` under
+    # pipefail, with dig exiting non-zero. No message, no container.
+    cat > "$BATS_TEST_TMPDIR/servfail" <<'EOF'
+#!/bin/bash
+echo ";; connection timed out; no servers could be reached" >&2
+exit 9
+EOF
+    run run_resolve "$BATS_TEST_TMPDIR/servfail"
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"attempt 1/3: dig exited 9"* ]]
+    [[ "$output" == *"attempt 3/3: dig exited 9"* ]]
+    [[ "$output" == *"no servers could be reached"* ]]
+}
+
+@test "an answer with no A records is distinguished from a failing lookup" {
+    # A CNAME-only answer, or one dnsmasq's stop-dns-rebind refused. Both look
+    # like an empty A list, and the raw answer is what tells them apart.
+    cat > "$BATS_TEST_TMPDIR/cname" <<'EOF'
+#!/bin/bash
+echo "release-assets.githubusercontent.com. 60 IN CNAME example.map.fastly.net."
+EOF
+    run run_resolve "$BATS_TEST_TMPDIR/cname" release-assets.githubusercontent.com
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"no A records"* ]]
+    [[ "$output" == *"CNAME"* ]]
+    [[ "$output" != *"dig exited"* ]]
+}
+
+@test "domain resolution recovers when a later attempt answers" {
+    cat > "$BATS_TEST_TMPDIR/flaky" <<'EOF'
+#!/bin/bash
+n="$BATS_TEST_TMPDIR/digs"
+count=$(cat "$n" 2>/dev/null || echo 0)
+echo $((count + 1)) > "$n"
+if [ "$count" -lt 1 ]; then exit 9; fi
+echo "files.pythonhosted.org.	60	IN	A	151.101.64.223"
+EOF
+    run run_resolve "$BATS_TEST_TMPDIR/flaky"
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"attempt 1/3: dig exited 9"* ]]
+    [[ "$output" == *"151.101.64.223"* ]]
+}
+
+@test "each dig attempt is bounded, so one unanswered lookup cannot eat the budget" {
+    # dig's defaults are 5s x 3 tries. Three attempts of that is 45s spent on
+    # one domain, inside a startup the test suite gives 120s in total.
+    grep -qE 'dig .*\+time=[0-9]+ \+tries=[0-9]+' "$SANDBOX_DIR/init-firewall.sh"
+}
+
+@test "the extra-domains loop resolves through the same path, so non-fatal is true" {
+    # Its comment says failures there are non-fatal. With a bare
+    # `ips=$(dig ... | awk ...)` that held for a domain that resolved to
+    # nothing and not for one whose lookup errored - pipefail took the whole
+    # script down instead of warning and skipping.
+    local bare
+    bare=$(awk '
+        /^[[:space:]]*#/ { next }
+        /^[[:space:]]*ips=\$\(dig/ { print FNR": "$0 }
+    ' "$SANDBOX_DIR/init-firewall.sh")
+    [ -z "$bare" ] || {
+        echo "a dig is still assigned directly, so pipefail can kill the script:" >&2
+        echo "$bare" >&2
+        false
+    }
+    grep -q 'if ! ips=$(resolve_domain "$domain"); then' "$SANDBOX_DIR/init-firewall.sh"
+}
+
+@test "an empty response is reported as empty, not as a missing field" {
+    cat > "$BATS_TEST_TMPDIR/empty" <<'EOF'
+#!/bin/bash
+exit 0
+EOF
+    run run_fetch "$BATS_TEST_TMPDIR/empty"
+    [ "$status" -ne 0 ]
+    [[ "$output" == *"empty response"* ]]
+}
+
 @test "the zsh installer runs from a file, so a failed download fails the build" {
     # Upstream documents `sh -c "$(wget -O- ...)"`. In that form a failed
     # download substitutes to an empty string and `sh -c ""` exits 0, so the
@@ -448,7 +740,12 @@ setup() {
     # Unauthenticated /meta is capped at 60 requests/hour per source IP,
     # shared across every container start; exhausting it fails the launch.
     grep -q 'gh_auth=(-H "Authorization: Bearer \$GH_TOKEN")' "$SANDBOX_DIR/init-firewall.sh"
-    grep -q 'curl -s \${gh_auth\[@\]+"\${gh_auth\[@\]}"} https://api.github.com/meta' "$SANDBOX_DIR/init-firewall.sh"
+    # The array reaching the request to /meta is the property; the flags around
+    # it are no longer pinned here, because they now have tests of their own
+    # ("every fetch in the container's startup path has a deadline" and the
+    # behavioural ones above). Pinning the whole invocation twice made adding a
+    # timeout to it fail a test about the token.
+    grep -q '\${gh_auth\[@\]+"\${gh_auth\[@\]}"} https://api.github.com/meta' "$SANDBOX_DIR/init-firewall.sh"
 }
 
 @test "ALLOW_GITHUB_PUSH defaults to 0 everywhere it's read" {
@@ -870,11 +1167,24 @@ this line is not json'
     # the container dies mid-startup - which used to surface as flaky,
     # sometimes vacuous tests rather than an obvious failure. See runtime.bats'
     # wait_ready comment.
+    # Detached and foreground starts both count - the transport test runs its
+    # container in the foreground, to read the container's own streams rather
+    # than the daemon's log driver (see its comment), and it needs a network
+    # every bit as much as the detached ones.
+    #
+    # Exempted: runs that replace the entrypoint with a plain shell. Those
+    # never reach dns-proxy.sh, so there is nothing for a resolver to be
+    # missing from - the image/checkout comparison in setup_file is one, and it
+    # only runs md5sum.
     local runs networks
-    runs=$(grep -cE '^\s*docker run -d' "$REPO_ROOT/test/runtime.bats")
+    runs=$(grep -E '^\s*(run timeout [0-9]+ )?docker run ' "$REPO_ROOT/test/runtime.bats" \
+        | grep -vc -- '--entrypoint sh')
     networks=$(grep -cE '^\s*--network ' "$REPO_ROOT/test/runtime.bats")
     [ "$runs" -gt 0 ]
-    [ "$runs" -eq "$networks" ]
+    [ "$runs" -eq "$networks" ] || {
+        echo "$runs container starts that run the real entrypoint, but $networks --network flags" >&2
+        false
+    }
 }
 
 @test "runtime.bats readiness waits for the final sleep infinity, not any startup sleep" {