#!/usr/bin/env bash
# SPDX-License-Identifier: GPL-3.0-or-later
# Copyright (C) 2026 Brad Anderson
#
# Take a fresh Debian install to a hardened CI runner host.
#
# Runs INSIDE the VM. The firewall that keeps this machine off your LAN is not
# here - it is on the Proxmox node, applied to this VM's tap device, because
# the premise of the privileged tier is that a compromised job reaches root
# here, and root can flush anything it can see. See proxmox/README.md.
#
# What this does install is real, and defends against a compromised job that
# has NOT escaped its container - which is the ordinary case, and the one the
# job-network allowlist exists for.
#
# Idempotent: re-running changes only what differs.

set -euo pipefail

PROG=${0##*/}
SRC="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/.." && pwd)"
ROLE=""
MGMT=""
SKIP_HARDENER=0
STATIC_IP=0
PINNED=0
NONINTERACTIVE=0
[ -t 0 ] || NONINTERACTIVE=1

usage() {
    cat >&2 <<EOF
Usage: sudo $PROG --role confined|privileged [--mgmt IP] [--skip-hardener]

  --role confined    jobs are confined on this host: no Docker socket in job
                     containers, own bridge network, deny-by-default egress
  --role privileged  jobs get the socket and the host network namespace, so
                     nothing on this host can confine them. Treat it as lost;
                     the boundary is the hypervisor's firewall
  --mgmt IP          restrict sshd to this source as well as the node firewall
  --skip-hardener    do the host hardening only, not the allowlist install.
                     Implied by --role privileged, where the allowlist has
                     nothing to attach to
  --static-ip        pin this host to the address, gateway and resolver it
                     currently has, so they stop depending on a DHCP lease.
                     Offered anyway when the interface is on DHCP and there is
                     a terminal to ask on
  --non-interactive  never prompt; take what the flags say and no more
EOF
    exit 2
}

while [ $# -gt 0 ]; do
    case "$1" in
        --role) ROLE="${2:?}"; shift ;;
        --mgmt) MGMT="${2:?}"; shift ;;
        --skip-hardener) SKIP_HARDENER=1 ;;
        --static-ip) STATIC_IP=1 ;;
        -n|--non-interactive) NONINTERACTIVE=1 ;;
        -h|--help) usage ;;
        *) echo "$PROG: unknown argument: $1" >&2; usage ;;
    esac
    shift
done
# `ci` and `runtime` were the original spellings. They named the tiers after
# what dejima used them for rather than after what a job on them is granted,
# which is the thing that has to be legible to anyone else. Still accepted,
# and said out loud rather than translated silently.
case "$ROLE" in
    confined|privileged) ;;
    ci)      echo "$PROG: --role ci is now --role confined" >&2;      ROLE=confined ;;
    runtime) echo "$PROG: --role runtime is now --role privileged" >&2; ROLE=privileged ;;
    *) usage ;;
esac

die() { echo "$PROG: $*" >&2; exit 1; }
say() { printf '%s\n' "$*" >&2; }
[ "$(id -u)" -eq 0 ] || die "run as root"

say "=============================================================="
say " dejima CI runner bootstrap - role: $ROLE"
say "=============================================================="
say

# --- packages ---------------------------------------------------------------
# docker.io from Debian rather than Docker's own repository: one less
# third-party apt source on a machine whose whole point is a small, known
# surface, and the daemon's version matters less here than what it is allowed
# to do.
# Resolution first, because everything below needs it and apt's failure for a
# host with no resolver does not mention DNS at all - it reports that it could
# not reach the mirrors, which reads as the network being down rather than as
# one missing line in one file.
#
# The case worth naming: a host converted to static addressing while a DHCP
# client still owns /etc/resolv.conf. The client has no lease to work from, so
# it rewrites the file with comments and no nameserver, and every symptom
# afterwards names something other than DNS.
say "Checking this host can resolve..."
if ! getent hosts deb.debian.org >/dev/null 2>&1; then
    {
        printf '%s: this host cannot resolve deb.debian.org, so nothing below\n' "$PROG"
        printf 'would work. Fix DNS first; none of this is installed yet.\n\n'
        printf 'What it is configured to use:\n'
        if [ -s /etc/resolv.conf ]; then
            sed 's/^/  /' /etc/resolv.conf
        else
            printf '  /etc/resolv.conf is empty or absent\n'
        fi
        printf '\n'
        if ! grep -q '^[[:space:]]*nameserver' /etc/resolv.conf 2>/dev/null; then
            printf 'There is no nameserver line at all. If the first line says it was\n'
            printf 'generated by a DHCP client and this host is statically addressed,\n'
            printf 'that client rewrote the file with nothing in it - it has no lease\n'
            printf 'to take servers from. Put one back, and stop it happening again:\n\n'
            printf '  printf %s | sudo tee /etc/resolv.conf.head\n' "'nameserver <ip>\\n'"
            printf '  systemctl disable --now dhcpcd\n'
            printf '  printf %s | sudo tee /etc/resolv.conf\n\n' "'nameserver <ip>\\n'"
            printf 'See proxmox/README.md section 2, "give it a stable address".\n'
        else
            printf 'A nameserver is configured, so it is not answering rather than\n'
            printf 'missing. Check it is reachable from here, and that the node\n'
            printf 'firewall permits udp/53 AND tcp/53 to that exact address - the\n'
            printf 'gateway is the usual wrong answer. See proxmox/README.md section 3.\n'
        fi
    } >&2
    exit 1
fi

# --- addressing -------------------------------------------------------------
# Offered before anything is installed, because the values it writes are read
# off the running interface and this is the moment they are known good: the
# host resolved a name a line ago, so its address, route and resolver are all
# working. Doing it later means reading them back through whatever the rest of
# this script has changed.
#
# Skipped silently unless the interface is actually on DHCP, so re-running this
# on a pinned host says nothing about it.
if [ "$STATIC_IP" -ne 0 ] || {
       [ "$NONINTERACTIVE" -eq 0 ] \
       && iface=$(ip route show default 2>/dev/null | awk '{print $5; exit}') \
       && [ -n "${iface:-}" ] \
       && grep -qE "^[[:space:]]*iface[[:space:]]+$iface[[:space:]]+inet[[:space:]]+dhcp" \
                /etc/network/interfaces 2>/dev/null
   }; then
    if [ "$STATIC_IP" -eq 0 ]; then
        say
        say "$iface is configured by DHCP. A runner wants an address that stays"
        say "put: you reach this machine by it for the life of the VM, and a"
        say "lease that moves is a machine you have to go and find."
        say
        say "This can pin it to exactly what it has now - same address, same"
        say "gateway, same resolver - so nothing about reachability changes and"
        say "there is nothing to type. Reserve the address on your DHCP server"
        say "afterwards, or put it outside the pool."
        say
        printf 'Pin this host to its current address? (Y/n) [y]: ' >&2
        IFS= read -r pin </dev/tty || pin=""
        case "${pin:-y}" in n|N|no|No|NO) pin="" ;; *) pin=y ;; esac
    else
        pin=y
    fi
    if [ -n "$pin" ]; then
        say
        if "$SRC/bin/dejima-ci-static-ip" \
               $( [ "$NONINTERACTIVE" -eq 1 ] && printf '%s' --non-interactive ); then
            PINNED=1
        else
            die "could not pin the address; nothing else has been changed yet"
        fi
    fi
fi

say "Installing packages..."
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
# docker-cli named explicitly alongside docker.io. Debian splits the client
# from the daemon, and --no-install-recommends - which is otherwise right on a
# machine whose value is a small surface - drops it. apt then reports success,
# the daemon is installed, and there is no `docker` binary at all.
#
# ipset and dnsmasq belong to install.sh, which only runs on the confined
# tier. The privileged tier's jobs are in the host network namespace, where a
# job network's resolver and its ipset have nothing to sit in front of - so on
# that role these install an unused listener on port 53, on the machine this
# same script strips listeners from forty lines below. iptables stays either
# way: the host's own egress policy uses it on both roles.
tier_packages=""
if [ "$ROLE" = confined ]; then
    tier_packages="ipset dnsmasq"
fi
# dbus named explicitly. systemd itself runs without it, but systemctl talks to
# the manager over the system bus for anything beyond a unit-file lookup - so
# on a host without it, everything below that disables a service, reloads sshd,
# runs daemon-reload or enables a unit fails with "Failed to connect to system
# scope bus" while `systemctl disable <absent unit>` still answers normally.
#
# It arrives as a recommendation of systemd rather than a dependency, which
# means --no-install-recommends here does not add it and an install that never
# had it, or a host where something removed it as an unused dependency, is
# indistinguishable from a working one until a unit needs managing.
# qemu-guest-agent, on a machine otherwise kept to a small surface, so the
# reasoning is worth stating.
#
# It listens on a virtio-serial channel to the hypervisor and on no network
# socket at all, and job containers have no access to /dev/virtio-ports, so it
# adds nothing reachable from a job or from the LAN. What it does add is
# node-to-guest authority - guest-exec and friends are root in here - and that
# direction is already total: the node owns this disk, can roll it back, power
# it off and attach a console. The agent grants the node nothing it lacks and
# grants this guest nothing at all.
#
# What it buys is three things the rest of this design already depends on: a
# filesystem freeze, so a snapshot of a running VM is consistent rather than
# equivalent to pulling the plug; the address in the hypervisor's summary,
# which is the first place anyone looks for a machine they cannot reach; and a
# shutdown that is negotiated rather than an ACPI event hopefully acted on -
# which matters because the rollback cadence runs `qm shutdown --forceStop 1`,
# and forceStop is the plug.
apt-get install -y -qq --no-install-recommends \
    docker.io docker-cli \
    dbus qemu-guest-agent \
    openssh-server \
    iptables $tier_packages \
    ca-certificates curl git jq \
    unattended-upgrades

# Verify what was just installed is usable, rather than trusting apt's exit
# code. It reported success for a set that contained no `docker` binary, and
# everything after this point assumes one - including the compose check, which
# then blamed the network for a client that was never there.
if ! command -v docker >/dev/null 2>&1; then
    {
        printf '%s: docker installed but there is no `docker` command.\n\n' "$PROG"
        printf 'What dpkg has:\n'
        dpkg -l 2>/dev/null | awk '/docker/ {printf "  %s %s %s\n", $1, $2, $3}'
        printf '\nDebian splits the client from the daemon. If docker.io is installed\n'
        printf 'and docker-cli is not, that is this - install it and re-run.\n'
    } >&2
    exit 1
fi
say "  docker     $(docker --version 2>&1 | head -1)"

# Said here because the failure is silent at both ends: with the option off the
# channel does not exist, the agent has nothing to attach to, and the
# hypervisor reports no address while snapshots of a running VM stay
# crash-consistent - with nothing anywhere saying why.
if [ -e /dev/virtio-ports/org.qemu.guest_agent.0 ]; then
    say "  agent      qemu-guest-agent, channel present"
else
    say "  agent      qemu-guest-agent installed, but this VM has no agent"
    say "             channel - the hypervisor's Qemu Agent option is off. Turn"
    say "             it on for consistent live snapshots and for this machine's"
    say "             address to appear in its summary."
fi

# Checked rather than assumed, for the same reason as docker above: apt
# reporting success is not the same as the thing working, and every remaining
# section of this script manages a unit.
if ! systemctl is-system-running >/dev/null 2>&1 \
   && ! systemctl list-units --type=service >/dev/null 2>&1; then
    {
        printf '%s: systemctl cannot reach the service manager.\n\n' "$PROG"
        printf 'PID 1 is %s. If that is systemd, the system bus is missing:\n' "$(ps -p 1 -o comm= 2>/dev/null)"
        printf '  dbus installed: %s\n' "$(dpkg -l dbus 2>/dev/null | grep -c '^ii')"
        printf '  socket:         %s\n\n' "$(ls /run/dbus/system_bus_socket 2>&1)"
        printf 'dbus was just installed, so this usually means the bus has not\n'
        printf 'started yet on a host that has been running without it. Restart\n'
        printf 'the machine and run this again - everything above is idempotent.\n\n'
        printf 'Nothing below this point would work: disabling a service,\n'
        printf 'reloading sshd, daemon-reload and enabling a unit all need it.\n'
    } >&2
    exit 1
fi

# Compose, by whatever route this release actually offers.
#
# The package is docker-compose-v2 from trixie onward and does not exist on
# bookworm, so naming it in the list above aborted the whole bootstrap on
# Debian 12. Detecting and shrugging is not much better: a runner host without
# compose cannot start a runner. So try each route that exists, in order of
# how much it adds to the machine, and stop at the first that works.
#
# v1 is not one of the routes. The compose files here use a top-level `name:`,
# which is compose-spec v2 and which the `docker-compose` package rejects -
# installing it would produce a working-looking host that fails at the first
# `docker compose up`.
have_compose() { docker compose version >/dev/null 2>&1; }
. /etc/os-release
codename="${VERSION_CODENAME:-}"

if have_compose; then
    say "  compose    already present"
else
    # 1. The distribution's own package, where it exists.
    if apt-cache show docker-compose-v2 2>/dev/null | grep -q '^Package:'; then
        say "  compose    installing docker-compose-v2 from $codename"
        apt-get install -y -qq --no-install-recommends docker-compose-v2 || true
    fi

    # 2. Backports, which is still Debian and still signed by Debian.
    if ! have_compose && [ -n "$codename" ]; then
        say "  compose    no distribution package on $codename; trying backports"
        printf 'deb http://deb.debian.org/debian %s-backports main\n' "$codename" \
            > /etc/apt/sources.list.d/backports.list
        apt-get update -qq || true
        apt-get install -y -qq --no-install-recommends \
            -t "$codename-backports" docker-compose-v2 || true
        if ! have_compose; then
            # Remove the source AND refresh, or apt keeps a list referring to
            # something that is no longer configured.
            rm -f /etc/apt/sources.list.d/backports.list
            apt-get update -qq || true
        fi
    fi

    # 3. Docker's own repository. Last because it is a third-party apt source
    #    on a machine whose value is a small, known surface - worth saying out
    #    loud rather than adding quietly.
    if ! have_compose; then
        say ""
        say "  compose    not available from Debian on $codename."
        say "             Adding Docker's official apt repository to get"
        say "             docker-compose-plugin. This is a third-party source on"
        say "             a host built to have few of them; the alternative is"
        say "             no compose, and no way to start a runner."
        say ""
        install -m 0755 -d /etc/apt/keyrings
        curl -fsSL https://download.docker.com/linux/debian/gpg \
            -o /etc/apt/keyrings/docker.asc || die "could not fetch Docker's signing key"
        chmod a+r /etc/apt/keyrings/docker.asc
        printf 'deb [arch=%s signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian %s stable\n' \
            "$(dpkg --print-architecture)" "$codename" \
            > /etc/apt/sources.list.d/docker.list
        apt-get update -qq
        apt-get install -y -qq --no-install-recommends docker-compose-plugin || true
    fi

    if ! have_compose; then
        {
            printf '%s: docker compose v2 is not usable.\n\n' "$PROG"
            printf 'What is on disk:\n'
            for d in /usr/libexec/docker/cli-plugins /usr/lib/docker/cli-plugins \
                     /usr/local/lib/docker/cli-plugins /usr/local/libexec/docker/cli-plugins; do
                [ -d "$d" ] && ls -1 "$d" 2>/dev/null | sed "s|^|  $d/|"
            done
            printf '  docker CLI: %s\n\n' "$(docker --version 2>&1 | head -1)"
            printf 'If a docker-compose binary is listed above then the package did\n'
            printf 'install, and the CLI searches these directories by default - so\n'
            printf 'report what you see rather than assuming a download failure. If\n'
            printf 'nothing is listed, no route produced one, which on a working network\n'
            printf 'usually means this host cannot reach download.docker.com.\n\n'
            printf 'Re-run once resolved; this script is idempotent and everything before\n'
            printf 'this point is already done.\n'
        } >&2
        exit 1
    fi
    say "  compose    $(docker compose version 2>/dev/null | head -1)"
fi

# --- services this host has no business running -----------------------------
# Every listener is a way in, and a CI runner needs none of these. avahi in
# particular announces the machine to the LAN it is meant to be isolated from.
say "Removing services a runner does not need..."
unwanted="avahi-daemon cups bluetooth modemmanager rpcbind"
# dnsmasq is not installed on a privileged host any more, but it was by
# earlier runs of this script, and an upgrade that leaves the old listener
# running is not an upgrade. Disabled rather than purged: someone co-locating
# both tiers on one machine still needs the package.
if [ "$ROLE" = privileged ]; then
    unwanted="$unwanted dnsmasq"
fi
for svc in $unwanted; do
    if systemctl list-unit-files "$svc.service" >/dev/null 2>&1 \
       && systemctl is-enabled "$svc" >/dev/null 2>&1; then
        systemctl disable --now "$svc" >/dev/null 2>&1 || true
        say "  disabled  $svc"
    fi
done
apt-get purge -y -qq avahi-daemon >/dev/null 2>&1 || true

# --- sshd -------------------------------------------------------------------
# Installed above as well as configured here. Configuring an sshd that is not
# present writes a file nothing reads, and on a VM whose only other way in is
# the hypervisor console that is a bad way to find out.
#
# Keys only, no root. The node firewall already restricts the source; this is
# the same restriction stated where a reader of this host will see it, and it
# survives the firewall being wrong.
say "Hardening sshd..."
sshd_drop=/etc/ssh/sshd_config.d/60-dejima-ci.conf
# Password authentication is the one setting here that can lock you out, and
# it does so silently: sshd keeps answering, the client is simply told that
# publickey is all that is on offer, and the log says "Connection closed by
# authenticating user" - which reads like a network fault rather than a
# decision this script made.
#
# So it is only applied where a key already exists to log in with. Checked
# across every home directory rather than only the invoking user's, because
# `sudo` from one account does not mean that is the account anyone SSHes in as.
have_authorized_key() {
    local f
    for f in /root/.ssh/authorized_keys /home/*/.ssh/authorized_keys; do
        [ -s "$f" ] || continue
        grep -qE '^[[:space:]]*[^#[:space:]]' "$f" && return 0
    done
    return 1
}

if have_authorized_key; then
    pw_auth="PasswordAuthentication no
KbdInteractiveAuthentication no"
else
    pw_auth="# PasswordAuthentication left ON: no authorized_keys was found on this
# host when $PROG ran, and turning it off would have left no way in but the
# hypervisor's console. Install a key and re-run this to have it disabled."
fi

new_sshd="# Written by $PROG.
PermitRootLogin no
$pw_auth
X11Forwarding no
AllowAgentForwarding no
AllowTcpForwarding no"
[ -z "$MGMT" ] || new_sshd="$new_sshd
# Belt and braces with the node firewall, which is the enforcing copy.
ListenAddress 0.0.0.0
Match Address $MGMT
    PermitTTY yes
Match all"
if [ -f "$sshd_drop" ] && [ "$new_sshd" = "$(cat "$sshd_drop")" ]; then
    say "  unchanged  $sshd_drop"
else
    install -d -m 0755 /etc/ssh/sshd_config.d
    printf '%s\n' "$new_sshd" > "$sshd_drop"
    sshd -t || die "sshd config rejected - not restarting, $sshd_drop is bad"
    systemctl reload ssh 2>/dev/null || systemctl reload sshd 2>/dev/null || true
    say "  written    $sshd_drop"
fi

if ! have_authorized_key; then
    say
    say "  PASSWORD AUTHENTICATION IS STILL ON, deliberately. No authorized_keys"
    say "  exists anywhere on this host, so disabling it would have left the"
    say "  hypervisor's console as the only way in."
    say
    say "  From the machine you administer this one from:"
    say
    say "    ssh-copy-id ${SUDO_USER:-<user>}@\$(hostname -I | awk '{print \$1}')"
    say
    say "  then re-run this script, which will disable it. Until then this host"
    say "  accepts a password from your management address - narrower than it"
    say "  sounds, since the node firewall admits only that address on 22, and"
    say "  wider than it should stay."

fi

# --- automatic security updates ---------------------------------------------
# A machine that exists to run untrusted code should not also be behind on
# kernel and container-runtime fixes.
say "Enabling unattended security upgrades..."
dpkg-reconfigure -f noninteractive unattended-upgrades >/dev/null 2>&1 || true

# --- the hardener itself ----------------------------------------------------
# Only for the confined tier, and the reason is the same one this whole
# arrangement rests on: the privileged tier's jobs run in the HOST network
# namespace. No bridge sits in front of them, so a job network, its resolver
# and its DOCKER-USER rules never see their traffic. Installing that machinery
# here would take a working evening and protect nothing, and - worse - would
# leave a privileged host that looks like it has an egress allowlist.
#
# What protects the privileged host is the node's firewall. That is the whole
# list.
if [ "$SKIP_HARDENER" -eq 1 ]; then
    say
    say "Skipping install.sh (--skip-hardener)."
elif [ "$ROLE" = privileged ]; then
    say
    say "Not running install.sh on a privileged host, deliberately."
    say
    say "Its jobs run in the host network namespace - that is the grant the tier"
    say "exists for - so no bridge is in front of them and a job network, its"
    say "resolver and its DOCKER-USER rules would never see their traffic."
    say "Installing them would leave this host looking like it has an egress"
    say "allowlist while having nothing of the sort."
    say
    say "What protects this host is the Proxmox node's firewall, and that is the"
    say "entire list. See proxmox/README.md."
    say
    say "If this machine also runs confined runners - not recommended, and not"
    say "what the tiering is for - run ./install.sh by hand."

    # The runner itself still has to be built, and on this role install.sh is
    # not there to do it. Install the generator and its templates directly, so
    # that this host ends up with the same "re-run it to change something"
    # story as a confined one rather than a hand-written compose file.
    install -m 0755 "$SRC/bin/dejima-ci-runners" /usr/local/sbin/dejima-ci-runners
    install -d -m 0755 /usr/local/share/dejima-ci/templates
    for tpl in "$SRC"/examples/*; do
        [ -f "$tpl" ] && install -m 0644 "$tpl" "/usr/local/share/dejima-ci/templates/${tpl##*/}"
    done

    # dejima-ci-upgrade, for the same reason and a stronger one.
    #
    # install.sh puts it on a confined host. Without this line a privileged host
    # never gets it from anywhere, because nothing else here installs anything -
    # so the tier whose ONLY boundary is the node firewall would be the one tier
    # with no way to ask which node-side versions it is missing, short of
    # cloning the repository every time.
    install -m 0755 "$SRC/bin/dejima-ci-upgrade" /usr/local/sbin/dejima-ci-upgrade
    install -d -m 0755 /usr/local/share/dejima-ci/upgrades
    for note in "$SRC"/upgrades/*.md; do
        [ -f "$note" ] && install -m 0644 "$note" "/usr/local/share/dejima-ci/upgrades/${note##*/}"
    done

    # And the tier, which is definitionally known here and is otherwise the
    # operator's to type. dejima-ci-upgrade refuses to guess it, on the grounds
    # that an absent config identifies a privileged host and a confined one
    # whose install.sh has not run - but this is the branch that IS the
    # privileged path, so there is nothing to guess.
    #
    # guest_version is deliberately NOT written. It tracks what install.sh
    # installed, and this tier runs none: the two programs placed above are
    # tools rather than controls, and a stale installed copy is a thing
    # dejima-ci-upgrade already reports about itself.
    install -d -m 0755 /etc/dejima-ci
    {
        [ -r /etc/dejima-ci/state ] && grep -v '^tier=' /etc/dejima-ci/state || true
        printf 'tier=privileged\n'
    } > /etc/dejima-ci/state.new
    sort -o /etc/dejima-ci/state.new /etc/dejima-ci/state.new
    chmod 0640 /etc/dejima-ci/state.new
    mv /etc/dejima-ci/state.new /etc/dejima-ci/state

    say
    say "  installed  /usr/local/sbin/dejima-ci-runners"
    say "  installed  /usr/local/sbin/dejima-ci-upgrade"
    say "  recorded   /etc/dejima-ci/state (tier=privileged)"

    # This runs under sudo, so its PATH is the one the operator's later sudo
    # commands get. Without a secure_path in sudoers, sudo inherits a non-root
    # PATH, and Debian leaves every sbin off that - so a correct install ends
    # with every documented command reporting "command not found".
    if [ "$(command -v dejima-ci-upgrade 2>/dev/null)" != /usr/local/sbin/dejima-ci-upgrade ]; then
        say
        say "  NOTE: /usr/local/sbin is not on the PATH sudo gives you here, so"
        say "  'sudo dejima-ci-upgrade ...' will not be found. Use the full path,"
        say "  or add a secure_path to sudoers that includes it."
    fi
    say
    /usr/local/sbin/dejima-ci-runners --role privileged || \
        say "  runner setup did not finish - re-run: sudo dejima-ci-runners --role privileged"
else
    say
    say "Running install.sh for the allowlist, resolver and job firewall..."
    say
    # Recorded rather than fatal. Dying here skips the summary below, and by
    # this point the host is most of the way configured - packages in, sshd
    # hardened, units installed, quite possibly runners registered and taking
    # jobs. An operator who is told only "install.sh did not complete" has to
    # work out for themselves which of those happened. The run still fails, at
    # the end, after saying what state the machine is in.
    HARDENER_FAILED=0
    "$SRC/install.sh" || HARDENER_FAILED=1
fi

# --- the VM's own egress ----------------------------------------------------
# Confined only, and for the same reason install.sh is confined only: on the
# privileged tier a job holds the Docker socket and runs in this host's network
# namespace, so it can flush an OUTPUT chain in one command. Installing one
# there would leave a host that looks like it restricts its own egress and does
# not - which is the failure this project argues against everywhere else, and
# it would be no less true here for being our own rule set.
#
# The node's firewall drops the same destinations and cannot be reached from
# the guest at all, so on that tier it is not a weaker copy of this: it is the
# only copy, and it is the real one.
if [ "$ROLE" = privileged ]; then
    if systemctl list-unit-files dejima-ci-host-egress.service >/dev/null 2>&1; then
        say
        say "Removing the host egress policy - a privileged job can flush it, so"
        say "it is not a control on this tier. The node's firewall is."
        systemctl disable --now dejima-ci-host-egress >/dev/null 2>&1 || true
        rm -f /etc/systemd/system/dejima-ci-host-egress.service \
              /usr/local/sbin/dejima-ci-host-egress
        systemctl daemon-reload
        say "  removed    dejima-ci-host-egress"
    fi
else
# Defence in depth, and labelled as such. A job that escapes its container
# reaches root here and can flush this in one command; what it cannot flush is
# the node's firewall. This exists for the case that does NOT escape, and to
# make an escape's first move visible rather than free.
say "Installing the host's own egress policy..."
cat > /etc/systemd/system/dejima-ci-host-egress.service <<'UNIT'
[Unit]
Description=dejima CI host egress policy (defence in depth; the node firewall is the boundary)
After=network-online.target docker.service
PartOf=docker.service
Wants=network-online.target

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/usr/local/sbin/dejima-ci-host-egress
# Without this, `systemctl stop` marks the unit inactive and leaves every rule
# in place - so the one command an operator reaches for when this policy has
# cut them off does nothing at all.
ExecStop=/usr/local/sbin/dejima-ci-host-egress --remove

[Install]
WantedBy=multi-user.target docker.service
UNIT

cat > /usr/local/sbin/dejima-ci-host-egress <<'EGRESS'
#!/usr/bin/env bash
# The VM's own outbound policy, in a chain this script owns.
#
# NOT the boundary. Root on this host can remove it; the Proxmox node's
# firewall is what actually keeps this machine off the LAN. This narrows what
# a contained process can reach and makes an escape's first outbound move
# something that had to step over a rule rather than something free.
set -euo pipefail
CHAIN=DEJIMA-HOST-OUT

# --remove takes the whole policy back off, which is what ExecStop calls. A
# firewall you cannot remove by the obvious means is one you cannot safely
# experiment with.
if [ "${1:-}" = --remove ]; then
    iptables -D OUTPUT -j "$CHAIN" 2>/dev/null || true
    iptables -F "$CHAIN" 2>/dev/null || true
    iptables -X "$CHAIN" 2>/dev/null || true
    echo "dejima-ci-host-egress: policy removed"
    exit 0
fi

iptables -N "$CHAIN" 2>/dev/null || true
iptables -F "$CHAIN"
iptables -C OUTPUT -j "$CHAIN" 2>/dev/null || iptables -I OUTPUT -j "$CHAIN"

# Anything already established, and the machine talking to itself.
iptables -A "$CHAIN" -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
iptables -A "$CHAIN" -o lo -j ACCEPT

# Docker's own bridges: container traffic is filtered in DOCKER-USER by
# dejima-ci-firewall, not here, and duplicating it would mean two places to
# keep in step.
iptables -A "$CHAIN" -o docker0 -j ACCEPT
for b in $(ip -o link show type bridge | awk -F': ' '{print $2}' | cut -d@ -f1); do
    iptables -A "$CHAIN" -o "$b" -j ACCEPT
done

# The LAN, minus whatever /etc/dejima-ci/host-egress-allow lists as
# host:port pairs. The node firewall enforces the same thing; a divergence
# between them is worth noticing, not worth relying on.
# This host's own resolvers, on 53, derived from the machine rather than copied
# into a second file by hand.
#
# They are on the LAN on almost every network, so without this the DROPs below
# reject every DNS query the host makes - and an ICMP admin-prohibited on a UDP
# socket is reported by Go, and so by Docker, as "connection refused". The
# symptom is `docker pull` failing against the registry with a connection error
# that names the registry and not the resolver, on a host that was working
# until this policy was enabled.
#
# Requiring the address in host-egress-allow instead would be one more pair of
# values that have to agree, of exactly the kind this project keeps warning
# about, with the same silence when they do not.
resolvers=$(awk '/^nameserver/ {print $2}' /etc/resolv.conf 2>/dev/null || true)
# systemd-resolved puts a loopback stub in resolv.conf and keeps the real
# servers to itself; loopback is already accepted above, but the queries the
# stub forwards leave this host and would be rejected.
if command -v resolvectl >/dev/null 2>&1; then
    resolvers="$resolvers $(resolvectl status 2>/dev/null | sed -n 's/.*DNS Servers: *//p' || true)"
fi
allowed_dns=0
for ns in $resolvers; do
    case "$ns" in *:*) continue ;; esac   # this chain is iptables, not ip6tables
    iptables -A "$CHAIN" -d "$ns" -p udp --dport 53 -j ACCEPT
    iptables -A "$CHAIN" -d "$ns" -p tcp --dport 53 -j ACCEPT
    echo "dejima-ci-host-egress: allowing DNS to $ns"
    allowed_dns=$((allowed_dns + 1))
done
if [ "$allowed_dns" -eq 0 ]; then
    # Said loudly because the alternative is silence. If this host resolves
    # through something on the LAN and nothing here found it, the rules below
    # reject every query it makes, and the first symptom is some unrelated
    # program reporting a connection refused by whatever it was trying to
    # reach - never by the resolver, and never naming this policy.
    echo "dejima-ci-host-egress: WARNING - found no nameserver to allow." >&2
    echo "dejima-ci-host-egress: /etc/resolv.conf named none, and resolvectl" >&2
    echo "dejima-ci-host-egress: either is absent or reported none. If this" >&2
    echo "dejima-ci-host-egress: host resolves through a LAN address, the" >&2
    echo "dejima-ci-host-egress: rules below now reject it: add it to" >&2
    echo "dejima-ci-host-egress: /etc/dejima-ci/host-egress-allow as '<ip> 53 udp'" >&2
    echo "dejima-ci-host-egress: and '<ip> 53 tcp', or run this with --remove." >&2
fi

if [ -r /etc/dejima-ci/host-egress-allow ]; then
    while read -r host port proto; do
        case "${host:-}" in ''|'#'*) continue ;; esac
        iptables -A "$CHAIN" -d "$host" -p "${proto:-tcp}" --dport "$port" -j ACCEPT
    done < /etc/dejima-ci/host-egress-allow
fi
for net in 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16; do
    iptables -A "$CHAIN" -d "$net" -j REJECT --reject-with icmp-admin-prohibited
done
EGRESS
chmod 755 /usr/local/sbin/dejima-ci-host-egress

install -d -m 0750 /etc/dejima-ci
if [ ! -f /etc/dejima-ci/host-egress-allow ]; then
    cat > /etc/dejima-ci/host-egress-allow <<'ALLOW'
# host port [proto] - LAN destinations this host may reach, beyond the ones it
# works out for itself. The Proxmox node's firewall enforces the same list;
# this copy is defence in depth.
#
# You do NOT need to list a resolver here. dejima-ci-host-egress reads this
# host's own nameservers from /etc/resolv.conf and allows them on 53 - if it
# did not, enabling the policy would stop the host resolving anything, and
# Docker reports a rejected DNS query as a connection refused by the registry
# rather than by the resolver.
#
# In practice that leaves the forge, which is reached by name and by address:
#
# <forge-ip> 443 tcp
ALLOW
    chmod 640 /etc/dejima-ci/host-egress-allow
    say "  created    /etc/dejima-ci/host-egress-allow - fill it in before enabling"
fi

systemctl daemon-reload
say "  installed  dejima-ci-host-egress.service (not enabled yet - see below)"

# --- fill it in and turn it on ----------------------------------------------
# The two destinations this host cannot work without are the resolver and the
# forge. The script derives the resolver from /etc/resolv.conf; the forge is
# derived here, from the URL install.sh just recorded, and resolved to
# addresses with the resolver that is about to be allowed.
#
# Asking for it instead would be one more value to transcribe, and the failure
# from getting it wrong is a host that cannot reach its forge - which presents
# as the runner going quiet rather than as a firewall rule.
forge_host=$(sed -n 's|^ *forge_api *= *||p' /etc/dejima-ci/config 2>/dev/null \
             | head -1 | sed 's|https\?://||; s|/.*||; s|:.*||')
forge_ips=""
[ -z "$forge_host" ] || forge_ips=$(getent ahostsv4 "$forge_host" 2>/dev/null \
                                    | awk '{print $1}' | sort -u | tr '\n' ' ')

if [ -n "$forge_ips" ]; then
    {
        echo "# Written by $PROG. host port [proto] - LAN destinations this host"
        echo "# may reach beyond the resolvers it works out for itself."
        echo "#"
        echo "# Derived from forge_api in /etc/dejima-ci/config: $forge_host"
        for ip in $forge_ips; do echo "$ip 443 tcp"; done
    } > /etc/dejima-ci/host-egress-allow
    chmod 640 /etc/dejima-ci/host-egress-allow
    say "  written    /etc/dejima-ci/host-egress-allow ($forge_host: $forge_ips)"

    systemctl enable --now dejima-ci-host-egress >/dev/null 2>&1 \
        && say "  enabled    dejima-ci-host-egress" \
        || say "  NOTE: could not enable dejima-ci-host-egress"

    # Verified rather than assumed, and rolled back if it does not hold. This
    # policy has exactly one way to go wrong and it takes the host off the
    # network, at which point the reason is no longer reachable either.
    if getent hosts "$forge_host" >/dev/null 2>&1; then
        say "  verified   $forge_host still resolves with the policy applied"
    else
        say
        say "  The policy is on and $forge_host no longer resolves, so it is"
        say "  removing something this host needs. Taking it back off."
        systemctl disable --now dejima-ci-host-egress >/dev/null 2>&1 || true
        say "  removed    dejima-ci-host-egress - fix the cause and re-run this"
    fi
else
    say "  NOTE: could not work out the forge's address, so"
    say "        /etc/dejima-ci/host-egress-allow is left as it is and the"
    say "        egress policy is NOT enabled. Enabling it without the forge in"
    say "        that file would cut this host off from it."
fi
fi


cat >&2 <<EOF

=============================================================
 What is left
=============================================================
EOF

if [ "${PINNED:-0}" -eq 1 ]; then
    cat >&2 <<'EOF'

0. This host's address, gateway and resolver were pinned to what it already
   had, so nothing about reaching it changed - but two things are left, and
   both are outside this machine:

     - RESERVE THAT ADDRESS on whatever hands out leases, or move it outside
       the pool. Until then the server is free to give it to something else,
       and the collision presents as two machines intermittently unreachable.
     - RESTART THIS VM once and check it comes back. The configuration takes
       effect at the next start, which is also the first time it is proven,
       and a snapshot taken before any of this makes that free to find out.
EOF
fi

cat >&2 <<EOF

1. If you skipped the runner setup above, or it did not finish, run it. It
   writes the compose project and the runner config together, so nothing has
   to be kept in step by hand:

     sudo dejima-ci-runners --role $ROLE

   It needs a registration token from your forge and names the exact page to
   mint one on. Grant it to as few repositories as you can stand.

2. RESTART this VM and check it comes back - address, route, name resolution,
   and 'systemctl is-system-running' answering rather than degraded. Nothing
   above has been through a boot yet, and a static network configuration takes
   effect at boot and not before.

3. Then, once step 1 shows the runner Online, shut this VM down and snapshot it
   - BEFORE the first job runs, and with 'Include RAM' UNTICKED:

     VM -> Snapshots -> Take Snapshot, named 'clean'

   Order matters, in four ways. Snapshot before registering and every rollback
   drops the runner off the forge. Snapshot after a job and you have preserved
   whatever worried you. Include RAM and the guest's clock comes back with it,
   so a rolled-back VM cannot verify a certificate. And snapshot before that
   restart and you have frozen a configuration nobody has tested - a rollback
   IS a cold boot, so the cadence would apply it unattended, every time.

4. Schedule the rollback on the NODE - nothing in here can be trusted to
   revert this VM, and a cadence you perform by hand is one you stop
   performing. Proxmox schedules backups, not rollbacks, so it is cron:

     /etc/cron.d/dejima-ci-rollback  on the Proxmox node

   proxmox/README.md section 4 has the line and what each part of it is for.
   Roll back after anything suspicious too; that one is not optional.

Remember what is and is not enforced here. The node's firewall keeps this
machine off your LAN and cannot be reached from inside it. Everything this
script installed can be removed by root on this host - which, on the
privileged role, is what a compromised job is assumed to get.
EOF

if [ "${HARDENER_FAILED:-0}" -ne 0 ]; then
    cat >&2 <<EOF

=============================================================
 install.sh did not finish
=============================================================

The host hardening above is done. What is not done is the job allowlist: no
server= lines have been rendered, so the resolver job containers are pointed
at answers for nothing, and any runner already registered will accept jobs
that then fail to resolve anything at all.

The reason was printed above, in install.sh's own output. The usual one is the
forge token: a token that is wrong rather than missing is only detectable by
asking the forge, and that is what fails.

Fix the cause, then run either of these - the first re-prompts for every
setting, the second only re-renders:

  sudo $SRC/install.sh
  sudo /usr/local/sbin/dejima-ci-allowlist render

Nothing needs undoing first. Both are idempotent.
EOF
    exit 1
fi
