dejima/dejima-ci-hardener
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request 'Deploy new programs without going through setup' (#28) from tools-only into main

Browse Source 
Reviewed-on: #28
Reviewed-by: branderson <brad@branderson.io>
This commit was merged in pull request #28.
This commit is contained in:
branderson
2026-09-02 16:16:05 -07:00
parent ae88b7767f aa1487ff64
commit b953973626
7 changed files with 465 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2
View File
@@ -219,6 +219,8 @@ sudo dejima-ci-allowlist show # what is installed right now
sudo dejima-ci-allowlist pools # the runner pools, and what each one reaches
sudo dejima-ci-allowlist verify # does the host agree with the pools file
sudo dejima-ci-firewall # reapply the egress rules
sudo ./install.sh --tools # new programs only, no setup questions
```
`verify` is the one to run after changing anything about a pool. Four things
bin/dejima-ci-allowlist
+138 -2
View File
@@ -708,6 +708,29 @@ V_UNKNOWN=0
v_ok() { printf ' ok %s\n' "$*" >&2; }
v_fail() { printf ' FAIL %s\n' "$*" >&2; V_FAIL=$((V_FAIL + 1)); }
v_unknown() { printf ' UNKNOWN %s\n' "$*" >&2; V_UNKNOWN=$((V_UNKNOWN + 1)); }
# Neither a pass nor a failure: something to tidy, which should not make an
# otherwise correct host report a problem. A verifier that cries about leftovers
# is one whose real findings get skimmed past.
v_note() { printf ' note %s\n' "$*" >&2; }
# The docker network sitting on a bridge, since the two are named independently
# and the pre-pools one is not called dejima-ci-<id> at all.
network_on_bridge() { # network_on_bridge iface -> network name, or empty
command -v docker >/dev/null 2>&1 || return 0
local n
while IFS= read -r n; do
[ -n "$n" ] || continue
[ "$(docker network inspect "$n" \
--format '{{index .Options "com.docker.network.bridge.name"}}' 2>/dev/null)" = "$1" ] \
&& { printf '%s\n' "$n"; return 0; }
done < <(docker network ls --format '{{.Name}}' 2>/dev/null || true)
return 0
}
attached_count() { # attached_count network -> how many containers, or empty
command -v docker >/dev/null 2>&1 || return 0
docker network inspect "$1" --format '{{len .Containers}}' 2>/dev/null || true
}
verify_network() { # verify_network id bridge
local id=$1 br=$2 net="dejima-ci-$id" actual
@@ -791,11 +814,40 @@ verify_set() { # verify_set name
fi
}
# The address dnsmasq answers on, and the one every pool's runners are given as
# --dns. It belongs to the shared network's bridge, not to any pool's, so it is
# the one thing the shared network is still for after every pool has its own.
#
# Checked first because losing it takes DNS away from every pool at once, and
# the symptom is jobs failing in apt rather than anything naming a resolver.
verify_gateway() {
local gw="${CFG[gateway]}"
[ -n "$gw" ] || { v_unknown "no gateway in $CONFIG, so nothing says where the resolver listens"; return; }
if ! command -v ip >/dev/null 2>&1; then
v_unknown "resolver address $gw: 'ip' is not on this host"; return
fi
local on; on=$(ip -o -4 addr show 2>/dev/null | awk -v g="$gw" '$4 ~ "^" g "/" { print $2; exit }')
if [ -n "$on" ]; then
v_ok "resolver address $gw is on $on"
else
v_fail "resolver address $gw is on no interface, so dnsmasq cannot answer"
say " Every pool's runners are given --dns $gw, so this is DNS"
say " gone for all of them at once. It belongs to the shared"
say " network's bridge. Recreate that network:"
say " docker network create --driver bridge \\"
say " -o com.docker.network.bridge.name=$(cfg_or_default bridge dejima-ci0) \\"
say " --subnet $(cfg_or_default subnet '<subnet>') --gateway $gw \\"
say " $(cfg_or_default network dejima-ci)"
say " sudo systemctl restart dnsmasq"
fi
}
cmd_verify() {
local f="${CFG[pools_file]}" any=0
[ -s "$f" ] || die "no pools declared in $f - nothing to verify.
sudo $PROG pools add <owner/repo | owner> [project-dir]"
verify_gateway
verify_set "${CFG[ipset]}-base"
say ""
@@ -812,6 +864,38 @@ cmd_verify() {
done < "$f"
[ "$any" = 1 ] || die "no usable lines in $f"
# THE DANGEROUS DIRECTION, checked first: a network that exists whose bridge
# has no rules. Every rule in DEJIMA-CI-FWD names an interface, so a bridge
# with none matches nothing, falls off the end of the chain and returns to
# Docker's default, which accepts. Anything landing there has unrestricted
# egress - a runner made without --pool, or a container left behind.
#
# Rules without a network are the harmless direction and are reported below.
# This is the one that fails open, and it is invisible: the network works,
# the jobs pass, and nothing is filtering them.
if command -v docker >/dev/null 2>&1 && command -v iptables >/dev/null 2>&1 \
&& iptables -S DEJIMA-CI-FWD >/dev/null 2>&1; then
local ruled n nb
ruled=$(iptables -S DEJIMA-CI-FWD 2>/dev/null \
| grep -oE -- '-i [^ ]+' | awk '{print $2}' | sort -u || true)
while IFS= read -r n; do
[ -n "$n" ] || continue
nb=$(docker network inspect "$n" \
--format '{{index .Options "com.docker.network.bridge.name"}}' 2>/dev/null || true)
[ -n "$nb" ] || continue
case "$nb" in "$(bridge_prefix)"*) ;; *) continue ;; esac
printf '%s\n' "$ruled" | grep -qx "$nb" && continue
say "unfiltered"
v_fail "$n is on $nb, which has no rules: anything on it reaches anything."
say " Every rule names an interface, so a bridge with none"
say " falls through to Docker's default, which accepts."
say " If it is the shared network, put its bridge back:"
say " set 'bridge = $nb' in $CONFIG"
say " sudo dejima-ci-firewall"
say ""
done < <(docker network ls --format '{{.Name}}' 2>/dev/null || true)
fi
# A bridge with rules that no pool claims. Almost always a pool removed from
# the file while its rules and its runners kept going, which is a set of
# jobs nothing here describes any more.
@@ -823,10 +907,62 @@ cmd_verify() {
| grep -E "^$(bridge_prefix)[0-9]+$" \
| grep -vxF -e "$claimed" || true) || true
if [ -n "$stray" ]; then
say "unclaimed"
# Not "unclaimed": the shared bridge lands here and is reported ok,
# under a heading calling it a leftover. A heading that disagrees
# with the line under it is read before the line is.
say "bridges no pool claims"
local configured=" $(cfg_or_default bridge '') "
while IFS= read -r b; do
[ -n "$b" ] || continue
v_fail "$b has rules but no pool in $f claims it"
local net att what
net=$(network_on_bridge "$b")
att=$(attached_count "${net:-}")
case "$configured" in
*" $b "*) what="the shared network from before pools existed" ;;
*) what="a pool that was removed while its rules stayed" ;;
esac
# The shared network holds the resolver address every pool
# depends on, so it is not retirable however empty it looks.
# Advising `docker network rm` on it took DNS away from every
# pool on a live host.
local holds_resolver=0
if [ -n "${CFG[gateway]}" ] && command -v ip >/dev/null 2>&1; then
[ "$(ip -o -4 addr show dev "$b" 2>/dev/null \
| awk -v g="${CFG[gateway]}" '$4 ~ "^" g "/" { print "y"; exit }')" = y ] \
&& holds_resolver=1
fi
if [ "$holds_resolver" = 1 ]; then
# Expected, and not something to tidy. It holds the address
# every pool resolves against, and its rules are what
# contains anything that lands on it - a runner made
# without --pool goes here. Suggesting either half be
# removed was wrong twice over.
v_ok "$b is the shared network: it holds the resolver at ${CFG[gateway]}, and its rules contain anything that lands on it"
elif [ "${att:-0}" -gt 0 ] 2>/dev/null; then
# Containers on a bridge no pool describes are jobs running
# under an allowlist nothing here decided. That is a finding.
v_fail "$b has ${att} container(s) and no pool claims it: $what."
say " Those jobs reach the base names only, since its rules"
say " name no pool's set. Move them onto their pool:"
say " sudo dejima-ci-runners --pool <scope>"
else
# Rules for a bridge nothing is on. Untidy rather than unsafe,
# so it does not fail the run.
v_note "$b carries rules but nothing is on it: $what."
say " Nothing reaches anything through it. To retire it:"
[ -n "$net" ] && say " docker network rm $net"
# Cleared, and said exactly. "Remove it from the bridge
# line" leaves an empty value, and the firewall used to
# refuse to start on one - a worse place to be than the
# leftover rules were.
say " set 'bridge =' in $CONFIG, with nothing after it"
say " sudo dejima-ci-firewall"
say " An empty bridge is fine once every pool has a network"
say " of its own: the pools supply the interfaces."
fi
done <<< "$stray"
say ""
fi
bin/dejima-ci-firewall
+42 -9
View File
@@ -75,21 +75,35 @@ BRIDGE=$(cfg_get bridge)
GATEWAY=$(cfg_get gateway)
SET=$(cfg_get ipset)
: "${SET:=ci-allow}"
[ -n "$BRIDGE" ] || die "bridge is not set in $CONFIG - the job network's bridge name, e.g. dejima-ci0"
# One or more, space-separated. The documented "a job network per runner" tier
# needs the same rules on each bridge, and the ipset is shared either way - it
# holds destinations, not per-bridge state.
# needs the same rules on each bridge, and the base set is shared either way.
#
# EMPTY IS ALLOWED, once pools exist. `bridge` names the shared network from
# before they did, and a host whose pools have all been given networks of their
# own has nothing left on it. Requiring it there meant the only way to retire
# that bridge was to leave a value naming an interface nothing uses, so the
# rules stayed and verify kept reporting it. Whether any bridge is known at all
# is checked below, once the pools file has had its say.
read -r -a BRIDGES <<< "$BRIDGE"
# Deferred until the pools file has been read, since either source may supply
# them and neither is required to on its own.
require_a_bridge() {
[ "${#BRIDGES[@]}" -gt 0 ] && return 0
die "no bridges to write rules for.
$CONFIG has no 'bridge', and $POOLS declares no pool. One of them has to name
an interface or there is nothing to apply rules to:
sudo dejima-ci-allowlist pools add <owner/repo | owner> [project-dir]
or set 'bridge' in $CONFIG to the shared network's interface."
}
# The bridge has to exist before rules can name it. Applied at boot by a unit
# ordered after docker.service, this is the case where the network has not
# been created yet - which is a setup step that was missed rather than a
# transient, so say which.
for b in "${BRIDGES[@]}"; do
ip link show "$b" >/dev/null 2>&1 \
|| die "no interface named $b - create the job network first, or correct 'bridge' in $CONFIG"
done
# A resolver that is bound but not answering is what a recreated bridge leaves
# behind, and it is invisible: `ss` still lists the socket, and every job dies
@@ -133,7 +147,10 @@ ipset create "${SET}-base" hash:net -exist
# The bridge name is derived, not configured, because it has to agree with what
# created the network. 15 characters is the kernel's limit on an interface name,
# which is why an id is a number rather than the scope it stands for.
BRIDGE_PREFIX="${BRIDGES[0]%%[0-9]*}"
# From the configured bridge when there is one, so an operator who named theirs
# something else keeps it. Otherwise the default, since with no shared bridge
# there is nothing to take the prefix from.
BRIDGE_PREFIX="${BRIDGES[0]:-}"; BRIDGE_PREFIX="${BRIDGE_PREFIX%%[0-9]*}"
: "${BRIDGE_PREFIX:=dejima-ci}"
declare -A BRIDGE_SET=()
@@ -152,6 +169,22 @@ if [ -r "$POOLS" ]; then
done < "$POOLS"
fi
require_a_bridge
# Every bridge, from either source. It used to run before the pools file was
# read, so a pool whose network had never been created got rules naming an
# interface that was not there - which iptables accepts, and which leaves that
# pool's jobs on a bridge with no rules at all.
for b in ${BRIDGES[@]+"${BRIDGES[@]}"}; do
ip link show "$b" >/dev/null 2>&1 \
|| die "no interface named $b - create its network first.
A pool's network is made by 'dejima-ci-allowlist pools add', and the shared
one by install.sh. If $b belongs to a pool that no longer exists, take it out
of $POOLS; if it is the shared network you have retired, clear 'bridge' in
$CONFIG."
done
# --- job egress: DOCKER-USER is the chain Docker jumps to first from FORWARD,
# --- and the only one whose contents Docker will not rewrite underneath you.
# Own a chain per table rather than appending into shared ones. This script
install.sh
+117 -66
View File
@@ -28,10 +28,12 @@ REPOS_FILE="$CONF_DIR/repos"
BASE_FILE="$CONF_DIR/base-domains"
NONINTERACTIVE=0
TOOLS_ONLY=0
[ -t 0 ] || NONINTERACTIVE=1
for a in "$@"; do
case "$a" in
-n|--non-interactive) NONINTERACTIVE=1 ;;
--tools) TOOLS_ONLY=1 ;;
-h|--help)
sed -n '2,10p' "$0" | sed 's/^# \{0,1\}//'
echo
@@ -50,6 +52,8 @@ writable() { mkdir -p "$1" 2>/dev/null || true; [ -d "$1" ] && [ -w "$1" ]; }
for d in "$CONF_DIR" "$BIN_DIR"; do
writable "$d" || die "cannot write $d - run as root, or set CONF_DIR/BIN_DIR/UNIT_DIR"
done
INSTALL_UNITS=1
writable "$UNIT_DIR" || INSTALL_UNITS=0
@@ -69,6 +73,81 @@ ask() {
# Only writes when the content differs, so a no-change run reports as such and
# leaves mtimes alone.
# --- the files this repository installs --------------------------------------
#
# A function because --tools runs it alone. Most upgrades change only the
# programs, and going through the whole setup to deploy them means answering
# prompts about a network, a resolver and a token that are already right - which
# is both tedious and a chance to change one of them by accident.
deploy_files() {
for prog in dejima-ci-allowlist dejima-ci-firewall dejima-ci-runners dejima-ci-static-ip \
dejima-ci-upgrade; do
if cmp -s "$SRC/bin/$prog" "$BIN_DIR/$prog" 2>/dev/null; then
say " unchanged $BIN_DIR/$prog"
else
install -m 0755 "$SRC/bin/$prog" "$BIN_DIR/$prog"
say " installed $BIN_DIR/$prog"
fi
done
# The templates dejima-ci-runners renders. Installed rather than read from the
# checkout, because it is installed too and has to keep working once whatever
# clone it came from is gone.
if writable "$TEMPLATE_DIR"; then
for tpl in "$SRC"/examples/*; do
[ -f "$tpl" ] || continue
if cmp -s "$tpl" "$TEMPLATE_DIR/${tpl##*/}" 2>/dev/null; then
say " unchanged $TEMPLATE_DIR/${tpl##*/}"
else
install -m 0644 "$tpl" "$TEMPLATE_DIR/${tpl##*/}"
say " installed $TEMPLATE_DIR/${tpl##*/}"
fi
done
else
say " skipped $TEMPLATE_DIR (not writable) - dejima-ci-runners will read"
say " them from this checkout instead"
fi
# The upgrade notes, so an operator on a runner with no checkout can still read
# what an upgrade asks of them. The installed copy knows only the versions that
# existed when it ran, and dejima-ci-upgrade says so rather than reporting "up
# to date" about versions it has never heard of.
if writable "$SHARE_DIR/upgrades"; then
for note in "$SRC"/upgrades/*.md; do
[ -f "$note" ] || continue
if cmp -s "$note" "$SHARE_DIR/upgrades/${note##*/}" 2>/dev/null; then
say " unchanged $SHARE_DIR/upgrades/${note##*/}"
else
install -m 0644 "$note" "$SHARE_DIR/upgrades/${note##*/}"
say " installed $SHARE_DIR/upgrades/${note##*/}"
fi
done
fi
# What this host is now at, guest-side, and which tier it is.
#
# Both are recorded rather than asked or detected. The version comes from the
# checkout's VERSION, so it records something this program just did. The tier is
# 'confined' by construction: install.sh does not run on a privileged host, and
# the whole point of that tier is that nothing here is installed there.
#
# THIS FILE IS INSIDE THE SNAPSHOT. On a runner that is correct rather than
# unfortunate - forget to re-snapshot after a guest-side upgrade and the marker
# rolls back alongside the change it was recording, so dejima-ci-upgrade goes
# back to reporting the upgrade as outstanding. It fails towards saying there is
# work to do.
version=$(tr -d '[:space:]' < "$SRC/VERSION" 2>/dev/null || true)
[ -n "$version" ] || die "no version in $SRC/VERSION - refusing to record an empty one"
{
[ -r "$CONF_DIR/state" ] && grep -vE '^(guest_version|tier)=' "$CONF_DIR/state" || true
printf 'guest_version=%s\ntier=confined\n' "$version"
} > "$CONF_DIR/state.new"
sort -o "$CONF_DIR/state.new" "$CONF_DIR/state.new"
chmod 0640 "$CONF_DIR/state.new"
mv "$CONF_DIR/state.new" "$CONF_DIR/state"
say " recorded $CONF_DIR/state (guest_version=$version, tier=confined)"
}
write_if_changed() {
local path="$1" mode="$2" content="$3"
if [ -f "$path" ] && [ "$content" = "$(cat "$path")" ]; then
@@ -87,6 +166,43 @@ write_if_changed() {
say " written $path"
}
# --tools: deploy the programs and stop.
#
# Most upgrades change only what is in bin/, and running the whole installer to
# put them there means answering prompts about a network, a resolver and a token
# that are already correct. Tedious, and a chance to change one of them by
# accident - which is how a re-run for one reason becomes a re-registration of
# something else.
#
# It writes no config, creates no network, touches no resolver, no firewall and
# no runner. It does record the guest version, because installing these programs
# is what guest_version is a record of.
if [ "$TOOLS_ONLY" = 1 ]; then
# A host that has never been set up gets programs with nothing to read. The
# programs would each say so on their own, but saying it here is the
# difference between an installer that looks like it worked and one that
# says what it did.
if [ ! -r "$CONF_DIR/config" ]; then
say "There is no $CONF_DIR/config on this host, so nothing here is set up"
say "yet, and --tools installs programs that have nothing to read."
say
say "Run the installer without --tools first."
exit 1
fi
say "=============================================================="
say " Tools only"
say "=============================================================="
say
deploy_files
say
say "Programs only. The config, the network, the resolver, the firewall and"
say "the runners are untouched, and no prompt above changed any of them."
say
say "If an upgrade note asks for anything besides new programs, run this"
say "again without --tools."
exit 0
fi
# --- existing values become defaults ----------------------------------------
cfg_get() {
local key="$1" line
@@ -690,72 +806,7 @@ out = $out
ref = $ref
reload = $reload"
for prog in dejima-ci-allowlist dejima-ci-firewall dejima-ci-runners dejima-ci-static-ip \
dejima-ci-upgrade; do
if cmp -s "$SRC/bin/$prog" "$BIN_DIR/$prog" 2>/dev/null; then
say " unchanged $BIN_DIR/$prog"
else
install -m 0755 "$SRC/bin/$prog" "$BIN_DIR/$prog"
say " installed $BIN_DIR/$prog"
fi
done
# The templates dejima-ci-runners renders. Installed rather than read from the
# checkout, because it is installed too and has to keep working once whatever
# clone it came from is gone.
if writable "$TEMPLATE_DIR"; then
for tpl in "$SRC"/examples/*; do
[ -f "$tpl" ] || continue
if cmp -s "$tpl" "$TEMPLATE_DIR/${tpl##*/}" 2>/dev/null; then
say " unchanged $TEMPLATE_DIR/${tpl##*/}"
else
install -m 0644 "$tpl" "$TEMPLATE_DIR/${tpl##*/}"
say " installed $TEMPLATE_DIR/${tpl##*/}"
fi
done
else
say " skipped $TEMPLATE_DIR (not writable) - dejima-ci-runners will read"
say " them from this checkout instead"
fi
# The upgrade notes, so an operator on a runner with no checkout can still read
# what an upgrade asks of them. The installed copy knows only the versions that
# existed when it ran, and dejima-ci-upgrade says so rather than reporting "up
# to date" about versions it has never heard of.
if writable "$SHARE_DIR/upgrades"; then
for note in "$SRC"/upgrades/*.md; do
[ -f "$note" ] || continue
if cmp -s "$note" "$SHARE_DIR/upgrades/${note##*/}" 2>/dev/null; then
say " unchanged $SHARE_DIR/upgrades/${note##*/}"
else
install -m 0644 "$note" "$SHARE_DIR/upgrades/${note##*/}"
say " installed $SHARE_DIR/upgrades/${note##*/}"
fi
done
fi
# What this host is now at, guest-side, and which tier it is.
#
# Both are recorded rather than asked or detected. The version comes from the
# checkout's VERSION, so it records something this program just did. The tier is
# 'confined' by construction: install.sh does not run on a privileged host, and
# the whole point of that tier is that nothing here is installed there.
#
# THIS FILE IS INSIDE THE SNAPSHOT. On a runner that is correct rather than
# unfortunate - forget to re-snapshot after a guest-side upgrade and the marker
# rolls back alongside the change it was recording, so dejima-ci-upgrade goes
# back to reporting the upgrade as outstanding. It fails towards saying there is
# work to do.
version=$(tr -d '[:space:]' < "$SRC/VERSION" 2>/dev/null || true)
[ -n "$version" ] || die "no version in $SRC/VERSION - refusing to record an empty one"
{
[ -r "$CONF_DIR/state" ] && grep -vE '^(guest_version|tier)=' "$CONF_DIR/state" || true
printf 'guest_version=%s\ntier=confined\n' "$version"
} > "$CONF_DIR/state.new"
sort -o "$CONF_DIR/state.new" "$CONF_DIR/state.new"
chmod 0640 "$CONF_DIR/state.new"
mv "$CONF_DIR/state.new" "$CONF_DIR/state"
say " recorded $CONF_DIR/state (guest_version=$version, tier=confined)"
deploy_files
# The resolver's static half. Its generated half - the server= lines - is
# written by dejima-ci-allowlist from origin/main, into a separate file in the
test/allowlist.bats
+123 -4
View File
@@ -972,6 +972,20 @@ DOCKEREOF
# getting it wrong looks like it worked.
verify_stubs() { # verify_stubs [rules-file]
# The resolver's address, on the shared bridge. Losing it takes DNS from
# every pool at once, so verify checks for it and the fixture has to have
# one for anything else to be exercised.
cat > "$T/stub/ip" <<'EOF2'
#!/usr/bin/env bash
case "$*" in
*"addr show dev dejima-ci0"*) [ -n "${STUB_NO_GW:-}" ] || echo "1: dejima-ci0 inet 192.0.2.1/24 scope global" ;;
*"addr show dev"*) ;;
*"addr show"*) [ -n "${STUB_NO_GW:-}" ] || echo "1: dejima-ci0 inet 192.0.2.1/24 scope global" ;;
esac
exit 0
EOF2
chmod +x "$T/stub/ip"
grep -q '^gateway' "$T/config" || echo "gateway = 192.0.2.1" >> "$T/config"
cat > "$T/stub/docker" <<'EOF2'
#!/usr/bin/env bash
case "$*" in *"network inspect dejima-ci-1"*) echo "dejima-ci1"; exit 0 ;; esac
@@ -1040,20 +1054,72 @@ EOF2
[[ "$output" == *"problem(s)"* ]]
}
@test "verify catches a bridge no pool claims" {
# Almost always a pool removed from the file while its rules and its
# runners kept going.
@test "a bridge with jobs on it and no pool is a failure" {
# Containers on a bridge no pool describes are jobs running under an
# allowlist nothing here decided.
verify_stubs
cat > "$T/stub/docker" <<'EOF2'
#!/usr/bin/env bash
case "$*" in
*"network ls"*) printf 'dejima-ci-9
' ;;
*"network inspect dejima-ci-1"*) echo "dejima-ci1" ;;
*'{{index .Options "com.docker.network.bridge.name"}}'*) echo "dejima-ci9" ;;
*"{{len .Containers}}"*) echo 3 ;;
*) exit 1 ;;
esac
exit 0
EOF2
chmod +x "$T/stub/docker"
printf -- '-A DEJIMA-CI-FWD -i dejima-ci9 -p tcp -m set --match-set ci-allow-9 dst -j ACCEPT\n' >> "$T/rules"
run_tool verify
[ "$status" -eq 1 ]
[[ "$output" == *"no pool in"* ]]
[[ "$output" == *"3 container(s) and no pool claims it"* ]]
[[ "$output" == *"--pool"* ]]
}
@test "a bridge with nothing on it is a note, not a failure" {
# Rules for a bridge nothing is on are untidy rather than unsafe. A verifier
# that cries about leftovers is one whose real findings get skimmed past.
verify_stubs
printf -- '-A DEJIMA-CI-FWD -i dejima-ci9 -p tcp -m set --match-set ci-allow-9 dst -j ACCEPT\n' >> "$T/rules"
run_tool verify
[ "$status" -eq 0 ]
[[ "$output" == *"nothing is on it"* ]]
[[ "$output" == *"dejima-ci-firewall"* ]]
}
@test "the shared bridge with rules is expected, not something to tidy" {
# It holds the address every pool resolves against, and its rules contain
# anything that lands on it. Suggesting either half be removed was wrong
# twice over: once for DNS, once for egress.
verify_stubs
echo "bridge = dejima-ci0" >> "$T/config"
printf -- '-A DEJIMA-CI-FWD -i dejima-ci0 -p tcp -m set --match-set ci-allow-base dst -j ACCEPT\n' >> "$T/rules"
run_tool verify
[ "$status" -eq 0 ]
[[ "$output" == *"is the shared network"* ]]
[[ "$output" != *"docker network rm"* ]]
[[ "$output" != *"set 'bridge ='"* ]]
}
@test "a resolver address on no interface is a failure, not a note" {
# Losing it takes DNS from every pool at once, and the symptom is jobs
# failing in apt rather than anything naming a resolver.
verify_stubs
STUB_NO_GW=1 run_tool verify
[ "$status" -eq 1 ]
[[ "$output" == *"cannot answer"* ]]
[[ "$output" == *"docker network create"* ]]
}
@test "verify refuses to pass what it could not check" {
# A verifier that reports success when it did not look is worse than no
# verifier, because it is the one that gets believed.
verify_stubs
# No gateway either, or the check for it runs with the real `ip` and reports
# a genuine failure, which is a different answer from "could not look".
sed -i '/^gateway /d' "$T/config"
# A PATH without /usr/sbin, rather than deleting the stubs: this container
# has a real iptables and a real ipset, so removing the stubs finds those
# instead of finding nothing.
@@ -1070,3 +1136,56 @@ EOF2
[ "$status" -eq 2 ]
[[ "$output" == *"no pools declared"* ]]
}
@test "install.sh --tools deploys programs without the setup flow" {
# Most upgrades change only what is in bin/, and running the whole installer
# to put them there means answering prompts about a network, a resolver and
# a token that are already correct.
inst="${BATS_TEST_DIRNAME}/../install.sh"
grep -q '\-\-tools) TOOLS_ONLY=1' "$inst"
grep -q '^deploy_files() {' "$inst"
}
@test "--tools on a host with no config refuses rather than half-installing" {
# The programs would have nothing to read, and an installer that looks like
# it worked is worse than one that says what it did not do.
grep -q 'no \$CONF_DIR/config on this host' "${BATS_TEST_DIRNAME}/../install.sh"
}
@test "a network whose bridge has no rules is a failure" {
# The direction that fails open. Every rule names an interface, so a bridge
# with none matches nothing, falls off the end of the chain and returns to
# Docker's default, which accepts. The network works and the jobs pass, and
# nothing is filtering them.
verify_stubs
cat > "$T/stub/docker" <<'EOF2'
#!/usr/bin/env bash
case "$*" in
*"network ls"*) printf 'dejima-ci
dejima-ci-1
' ;;
*"inspect dejima-ci-1 --format"*) echo dejima-ci1 ;;
*"inspect dejima-ci --format"*) echo dejima-ci0 ;;
*"len .Containers"*) echo 0 ;;
*) exit 1 ;;
esac
exit 0
EOF2
chmod +x "$T/stub/docker"
run_tool verify
[ "$status" -eq 1 ]
[[ "$output" == *"which has no rules"* ]]
[[ "$output" == *"reaches anything"* ]]
[[ "$output" == *"bridge = dejima-ci0"* ]]
}
@test "the heading does not call the shared bridge a leftover" {
# It is reported ok and described as expected, and sat under a heading
# saying "unclaimed". A heading that disagrees with the line under it is
# read before the line is.
verify_stubs
echo "bridge = dejima-ci0" >> "$T/config"
printf -- '-A DEJIMA-CI-FWD -i dejima-ci0 -p tcp -m set --match-set ci-allow-base dst -j ACCEPT\n' >> "$T/rules"
run_tool verify
[[ "$output" != *"unclaimed"* ]]
}
test/firewall.bats
+23
View File
@@ -86,3 +86,26 @@ setup() {
esac
done
}
@test "an empty bridge is allowed once the pools supply one" {
# `bridge` names the shared network from before pools existed, and a host
# whose pools all have networks of their own has nothing left on it.
# Requiring it meant the only way to retire that bridge was to leave a value
# naming an interface nothing uses, so the rules stayed.
grep -q 'EMPTY IS ALLOWED, once pools exist' "$BATS_TEST_DIRNAME/../bin/dejima-ci-firewall"
! grep -q 'bridge is not set in \$CONFIG' "$BATS_TEST_DIRNAME/../bin/dejima-ci-firewall"
}
@test "no bridge from either source is still refused" {
grep -q 'no bridges to write rules for' "$BATS_TEST_DIRNAME/../bin/dejima-ci-firewall"
}
@test "every bridge is checked for existence, including a pool's" {
# The check ran before the pools file was read, so a pool whose network was
# never created got rules naming an interface that is not there - which
# iptables accepts, leaving that pool's jobs on a bridge with no rules.
fw="$BATS_TEST_DIRNAME/../bin/dejima-ci-firewall"
pools_line=$(grep -n 'done < "$POOLS"' "$fw" | head -1 | cut -d: -f1)
check_line=$(grep -n 'ip link show "$b"' "$fw" | head -1 | cut -d: -f1)
[ "$check_line" -gt "$pools_line" ]
}
upgrades/0003-per-pool-sets.md
+20
View File
@@ -55,6 +55,11 @@ git pull
sudo ./install.sh
```
**Not `--tools` for this one.** That flag deploys the programs and stops, which
is right for an upgrade that changes only what is in `bin/`. This one is a full
run, because the questions about the network and the resolver are what let it
notice anything else that has moved.
**Say no to the runner question.** On a host that already has pools it now
defaults to no, and lists what you have with each pool's count, label and
network, so there is nothing to go and look up. Your runners are untouched
@@ -133,6 +138,21 @@ the jobs pass either way.
It exits 0 when they agree, 1 when they do not, and **2 when something could not
be checked**, which is not a pass. Run it on the runner host, with sudo.
**Leave the shared network alone.** Once every pool has one of its own it
carries no jobs, which makes it look retirable. It is not, for two reasons that
are easy to meet and hard to diagnose:
- the resolver listens on its gateway address, and every pool's runners are
given that address as `--dns`, so removing the network takes DNS from all of
them at once;
- every firewall rule names an interface, so clearing `bridge` leaves that one
with no rules. Anything that lands there - a runner made without `--pool`, a
container left behind - matches nothing, falls off the end of the chain, and
gets Docker's default, which accepts.
`verify` reports it as expected rather than as something to tidy, and fails if
either the resolver address or its rules go missing.
## What this does not do
**Every job can still resolve every allowlisted name.** One dnsmasq has one