From 4a4da08fa01bc81c4519985e4431d880ca6ced68 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Thu, 15 Oct 2026 17:55:25 -0700 Subject: [PATCH 1/2] Mirror main to GitHub and publish the image to GHCR mirror/github-mirror.sh publishes main and the tags on it to a read-only GitHub mirror, as one extra commit that adds a notice under the README title, so every forge commit is unchanged there and each sync is a fast-forward. A systemd user service and hourly timer run it from a host that can push to GitHub; --settings turns off issues, pull requests and the rest there. A GitHub Actions workflow builds the image on the mirror and pushes it to ghcr.io, freeing runner disk first since the CUDA build of torch is several GB. docker-compose.yml names the image after IMAGE_PREFIX and IMAGE_TAG, so setting them pulls the published image instead of building, and the README says how. --- .github/workflows/images.yml | 62 ++++++++++++ README.md | 16 +++ docker-compose.yml | 3 + mirror/README.md | 28 ++++++ mirror/github-mirror.sh | 133 +++++++++++++++++++++++++ mirror/songtrace-github-mirror.service | 26 +++++ mirror/songtrace-github-mirror.timer | 9 ++ 7 files changed, 277 insertions(+) create mode 100644 .github/workflows/images.yml create mode 100644 mirror/README.md create mode 100755 mirror/github-mirror.sh create mode 100644 mirror/songtrace-github-mirror.service create mode 100644 mirror/songtrace-github-mirror.timer diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml new file mode 100644 index 0000000..a33d718 --- /dev/null +++ b/.github/workflows/images.yml @@ -0,0 +1,62 @@ +# Builds the songtrace image on the GitHub mirror and pushes it to GitHub's +# container registry, under the repository's owner. Gitea ignores this +# directory while .gitea/workflows exists, so the forge does not run it. +name: Images + +on: + push: + branches: [main] + tags: ['v*.*.*'] + workflow_dispatch: + +permissions: + contents: read + packages: write + +jobs: + build: + runs-on: ubuntu-latest + steps: + # The image carries CUDA builds of torch, several GB of them, which is + # more than a hosted runner has free until its preinstalled toolchains + # are gone. + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/share/boost + sudo docker image prune --all --force + df -h / + + - uses: actions/checkout@v4 + + - name: Lowercase the owner + id: owner + shell: bash + run: echo "name=${GITHUB_REPOSITORY_OWNER,,}" >> "$GITHUB_OUTPUT" + + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ steps.owner.outputs.name }}/songtrace + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=ref,event=branch + + - uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64 + provenance: false diff --git a/README.md b/README.md index 9e3e5b1..5767ec1 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,18 @@ docker compose up -d Open `http://localhost:8000`, fill in the form and press **CREATE SONG**. +That builds the image here. To pull the published one instead, put this in a +`.env` file next to `docker-compose.yml` and run +`docker compose pull && docker compose up -d`: + +```sh +IMAGE_PREFIX=ghcr.io/branderson/ +IMAGE_TAG=main +``` + +`main` follows the `main` branch. Release tags will add `latest` for the +newest release and `1.2` for the newest of a minor version. + Finished songs go to `output_dir` (`/data/output` in the container). To write them straight into your game's song collection, mount that folder there in `docker-compose.yml`: @@ -212,6 +224,10 @@ pytest `pytest` also runs the web UI's JavaScript tests in `tests/js` when Node is installed. Pull requests run the full suite in CI. +songtrace is developed on a private forge and published to +[GitHub](https://github.com/branderson/songtrace) as a read-only mirror, which +also builds the published image; see `mirror/README.md`. + `scripts/screenshot_webui.py` regenerates `docs/images/webui.png` from a seeded copy of the app (`playwright install chromium` first). With `ffmpeg` on `PATH`, the shot includes the player. diff --git a/docker-compose.yml b/docker-compose.yml index cf17570..a24c77c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,9 @@ services: songtrace: + # built here by default; set IMAGE_PREFIX (and IMAGE_TAG) to pull a + # published image instead, see the README's quick start build: . + image: ${IMAGE_PREFIX:-}songtrace:${IMAGE_TAG:-latest} ports: - "8000:8000" volumes: diff --git a/mirror/README.md b/mirror/README.md new file mode 100644 index 0000000..6839a0a --- /dev/null +++ b/mirror/README.md @@ -0,0 +1,28 @@ +# GitHub mirror + +songtrace is developed on a private forge. `github-mirror.sh` publishes its +`main` and the tags on `main` to +[github.com/branderson/songtrace](https://github.com/branderson/songtrace), +hourly, from a host that can push to GitHub. Issues and pull requests are off +there. Actions stays on, since `.github/workflows/images.yml` builds the image +that `ghcr.io/branderson/songtrace` serves. + +The mirror's `main` is the forge's plus one commit, which adds a read-only notice +under the README's title. Every forge commit is in the mirror unchanged, and +each sync fast-forwards. If the README loses its `# ` title, the sync stops +instead of publishing without the notice. + +## Setup + +1. Set up the timer as described at the top of + `songtrace-github-mirror.service`. Give it a deploy key with write access to + the GitHub repository. +2. Run `github-mirror.sh --dry-run` to build what a sync would publish without + pushing it. +3. Once the first sync has pushed, run `github-mirror.sh --settings` (needs + `gh`). It turns off issues, pull requests, projects, wiki and discussions. +4. The first image build creates the package as private. In its settings on + GitHub, set its visibility to public. + +The timer runs a copy of the script, not the checkout's. Reinstall it after +reviewing a change to `mirror/`. diff --git a/mirror/github-mirror.sh b/mirror/github-mirror.sh new file mode 100755 index 0000000..2e158bb --- /dev/null +++ b/mirror/github-mirror.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Publishes main and its release tags to a read-only mirror on GitHub. +# +# MIRROR_SOURCE= mirror/github-mirror.sh sync +# ... mirror/github-mirror.sh --dry-run build and print, push nothing +# mirror/github-mirror.sh --settings configure the GitHub repo (needs gh) +# mirror/github-mirror.sh --transform DIR rewrite a checkout in place +# +# Runs on a host that can push to GitHub, never in a dejima session: a sandbox +# may push only to its own forge. +# +# The GitHub copy is not byte-for-byte. Its main is one extra commit on top of +# the forge's main, whose tree is the forge's with the rewrites in transform() +# applied, and whose parents are the forge commit and the previous mirror +# commit. So every forge SHA is in the mirror unchanged, and each sync is a +# fast-forward for anyone tracking it. Tags are pushed as they are, pointing at +# the untransformed forge commits. +# +# Environment: +# MIRROR_SOURCE forge clone URL (required to sync) +# MIRROR_REPO GitHub repository, default branderson/songtrace +# MIRROR_TARGET push URL, default git@github.com:$MIRROR_REPO.git +# MIRROR_STATE working bare repository, default ~/.cache/songtrace-github-mirror.git +set -euo pipefail + +MIRROR_REPO="${MIRROR_REPO:-branderson/songtrace}" +MIRROR_TARGET="${MIRROR_TARGET:-git@github.com:$MIRROR_REPO.git}" +MIRROR_STATE="${MIRROR_STATE:-${XDG_CACHE_HOME:-$HOME/.cache}/songtrace-github-mirror.git}" + +say() { echo "github-mirror: $*"; } +die() { say "$*" >&2; exit 1; } + +WORK="" +cleanup() { + [ -n "$WORK" ] || return 0 + git -C "$MIRROR_STATE" worktree remove --force "$WORK" 2>/dev/null || true + rm -rf "$WORK" +} + +# Rewrites a checkout for GitHub. Each rewrite is checked, so a README that +# drifted away from what this expects stops the mirror instead of publishing +# the forge's version. Running it twice changes nothing. +transform() { + local readme="$1/README.md" + # perl -i would write a symlink's target back as the README, publishing it. + [ -f "$readme" ] && [ ! -L "$readme" ] || die "README.md is missing or a symlink in $1" + + # The first thing a visitor to a mirror does is open an issue or a pull + # request. Under the title. + grep -q '^> \*\*This is a read-only mirror' "$readme" || perl -0pi -e ' + s{^(# [^\n]*)\n}{$1\n\n> **This is a read-only mirror.** songtrace is developed on a private forge,\n> and issues and pull requests are not accepted here.\n}m + ' "$readme" + grep -q '^> \*\*This is a read-only mirror' "$readme" \ + || die "README.md: no title to put the mirror notice under" +} + +# Issues, pull requests, wiki and the like off; Actions stays on, since it +# publishes the image. Settings rather than content, so a sync does not +# reapply them; run once after creating the repository. +settings() { + command -v gh >/dev/null || die "--settings needs the gh CLI" + gh api -X PATCH "repos/$MIRROR_REPO" --silent \ + -F has_issues=false -F has_pull_requests=false -F has_projects=false \ + -F has_wiki=false -F has_discussions=false +} + +sync() { + [ -n "${MIRROR_SOURCE:-}" ] || die "MIRROR_SOURCE is not set" + local git=(git -C "$MIRROR_STATE") + trap cleanup EXIT + [ -d "$MIRROR_STATE" ] || git init -q --bare "$MIRROR_STATE" + + "${git[@]}" fetch -q --no-tags --prune "$MIRROR_SOURCE" \ + '+refs/heads/main:refs/source/main' '+refs/tags/*:refs/source/tags/*' + local src prev="" + src="$("${git[@]}" rev-parse refs/source/main)" + + # An absent main (exit 2) is a first run; anything else is a real failure. + local rc=0 + git ls-remote -q --exit-code "$MIRROR_TARGET" refs/heads/main >/dev/null || rc=$? + if [ "$rc" -eq 0 ]; then + "${git[@]}" fetch -q --no-tags "$MIRROR_TARGET" '+refs/heads/main:refs/mirror/main' + prev="$("${git[@]}" rev-parse refs/mirror/main)" + elif [ "$rc" -ne 2 ]; then + die "cannot read $MIRROR_TARGET" + fi + + WORK="$(mktemp -d)" + "${git[@]}" worktree add -q --detach "$WORK" "$src" + transform "$WORK" + git -C "$WORK" add -A + local tree + tree="$(git -C "$WORK" write-tree)" + + if [ -n "$prev" ] && [ "$("${git[@]}" rev-parse "$prev^1")" = "$src" ] \ + && [ "$("${git[@]}" rev-parse "$prev^{tree}")" = "$tree" ]; then + say "already at ${src:0:12}" + else + # The forge commit's committer and date, so the same input always + # makes the same commit. + local name email date commit + IFS=$'\t' read -r name email date < <("${git[@]}" log -1 --format='%cn%x09%ce%x09%cI' "$src") + commit="$(GIT_AUTHOR_NAME="$name" GIT_AUTHOR_EMAIL="$email" GIT_AUTHOR_DATE="$date" \ + GIT_COMMITTER_NAME="$name" GIT_COMMITTER_EMAIL="$email" GIT_COMMITTER_DATE="$date" \ + "${git[@]}" commit-tree "$tree" -p "$src" ${prev:+-p "$prev"} \ + -m "Mirror ${src:0:12} for GitHub" \ + -m "README header added by mirror/github-mirror.sh.")" + "${git[@]}" update-ref refs/mirror/next "$commit" + [ -n "$DRY_RUN" ] || "${git[@]}" push -q "$MIRROR_TARGET" "$commit:refs/heads/main" + say "main is ${src:0:12} as ${commit:0:12}" + fi + + # Only tags on main: one on another branch would publish unreleased work. + local ref + local refspecs=() + while IFS= read -r ref; do + if "${git[@]}" merge-base --is-ancestor "$ref" "$src"; then + refspecs+=("$ref:refs/tags/${ref#refs/source/tags/}") + say "tag ${ref#refs/source/tags/}" + fi + done < <("${git[@]}" for-each-ref --format='%(refname)' refs/source/tags/) + [ "${#refspecs[@]}" -eq 0 ] || [ -n "$DRY_RUN" ] \ + || "${git[@]}" push -q "$MIRROR_TARGET" "${refspecs[@]}" +} + +DRY_RUN="" +case "${1:-}" in + "") sync ;; + --dry-run) DRY_RUN=1; sync ;; + --settings) settings ;; + --transform) [ -n "${2:-}" ] || die "--transform needs a directory"; transform "$2" ;; + *) sed -n '4,7p' "$0" >&2; exit 2 ;; +esac diff --git a/mirror/songtrace-github-mirror.service b/mirror/songtrace-github-mirror.service new file mode 100644 index 0000000..7fcdbdf --- /dev/null +++ b/mirror/songtrace-github-mirror.service @@ -0,0 +1,26 @@ +# User unit for the GitHub mirror, on a host that can push to GitHub. Never in +# a dejima session: a sandbox may push only to its own forge. +# +# install -m 755 mirror/github-mirror.sh ~/.local/bin/songtrace-github-mirror +# cp mirror/songtrace-github-mirror.service mirror/songtrace-github-mirror.timer ~/.config/systemd/user/ +# systemctl --user daemon-reload +# systemctl --user enable --now songtrace-github-mirror.timer +# +# A copy, not a link into the checkout: the timer holds a key that can push to +# GitHub, and a session working in this checkout can edit the script. +# Reinstall it after reviewing an update. +# +# ~/.config/songtrace/github-mirror.env sets MIRROR_SOURCE, the forge +# clone URL, and whatever else mirror/github-mirror.sh documents. The GitHub +# key, a deploy key with write access to the repository, belongs behind an ssh +# Host alias named in MIRROR_TARGET, so it is not also offered to the forge: +# +# MIRROR_SOURCE=git@forge.example.com:owner/songtrace.git +# MIRROR_TARGET=git@github-songtrace:branderson/songtrace.git +[Unit] +Description=Mirror songtrace's main and release tags to GitHub + +[Service] +Type=oneshot +EnvironmentFile=%h/.config/songtrace/github-mirror.env +ExecStart=%h/.local/bin/songtrace-github-mirror diff --git a/mirror/songtrace-github-mirror.timer b/mirror/songtrace-github-mirror.timer new file mode 100644 index 0000000..1ac7045 --- /dev/null +++ b/mirror/songtrace-github-mirror.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Mirror songtrace's main and release tags to GitHub, hourly + +[Timer] +OnCalendar=hourly +Persistent=true + +[Install] +WantedBy=timers.target -- 2.54.0 From 0fc4ad0e0884f0f903ed24a066666d457aa63fcd Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Fri, 16 Oct 2026 00:10:34 -0700 Subject: [PATCH 2/2] Drop the GHCR image build; users build with docker compose Publishing a prebuilt image can wait. Remove the GitHub Actions image workflow and the IMAGE_PREFIX/IMAGE_TAG image name in docker-compose.yml, and have the README's quick start build with docker compose up -d --build, which also rebuilds after a git pull. The mirror's docs and script no longer mention Actions or the package. --- .github/workflows/images.yml | 62 ------------------------------------ README.md | 21 +++--------- docker-compose.yml | 3 -- mirror/README.md | 5 +-- mirror/github-mirror.sh | 5 ++- 5 files changed, 8 insertions(+), 88 deletions(-) delete mode 100644 .github/workflows/images.yml diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml deleted file mode 100644 index a33d718..0000000 --- a/.github/workflows/images.yml +++ /dev/null @@ -1,62 +0,0 @@ -# Builds the songtrace image on the GitHub mirror and pushes it to GitHub's -# container registry, under the repository's owner. Gitea ignores this -# directory while .gitea/workflows exists, so the forge does not run it. -name: Images - -on: - push: - branches: [main] - tags: ['v*.*.*'] - workflow_dispatch: - -permissions: - contents: read - packages: write - -jobs: - build: - runs-on: ubuntu-latest - steps: - # The image carries CUDA builds of torch, several GB of them, which is - # more than a hosted runner has free until its preinstalled toolchains - # are gone. - - name: Free disk space - run: | - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ - /opt/hostedtoolcache/CodeQL /usr/local/share/boost - sudo docker image prune --all --force - df -h / - - - uses: actions/checkout@v4 - - - name: Lowercase the owner - id: owner - shell: bash - run: echo "name=${GITHUB_REPOSITORY_OWNER,,}" >> "$GITHUB_OUTPUT" - - - uses: docker/setup-buildx-action@v3 - - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - id: meta - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ steps.owner.outputs.name }}/songtrace - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=ref,event=branch - - - uses: docker/build-push-action@v6 - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64 - provenance: false diff --git a/README.md b/README.md index 5767ec1..c29aa37 100644 --- a/README.md +++ b/README.md @@ -53,22 +53,11 @@ what it got wrong before you sing it. ```sh cp config.example.toml config.toml $EDITOR config.toml # set whisper_url to your Whisper server -docker compose up -d +docker compose up -d --build ``` -Open `http://localhost:8000`, fill in the form and press **CREATE SONG**. - -That builds the image here. To pull the published one instead, put this in a -`.env` file next to `docker-compose.yml` and run -`docker compose pull && docker compose up -d`: - -```sh -IMAGE_PREFIX=ghcr.io/branderson/ -IMAGE_TAG=main -``` - -`main` follows the `main` branch. Release tags will add `latest` for the -newest release and `1.2` for the newest of a minor version. +Open `http://localhost:8000`, fill in the form and press **CREATE SONG**. To +update, `git pull` and run the same `docker compose` command again. Finished songs go to `output_dir` (`/data/output` in the container). To write them straight into your game's song collection, mount that folder there in @@ -225,8 +214,8 @@ pytest installed. Pull requests run the full suite in CI. songtrace is developed on a private forge and published to -[GitHub](https://github.com/branderson/songtrace) as a read-only mirror, which -also builds the published image; see `mirror/README.md`. +[GitHub](https://github.com/branderson/songtrace) as a read-only mirror; see +`mirror/README.md`. `scripts/screenshot_webui.py` regenerates `docs/images/webui.png` from a seeded copy of the app (`playwright install chromium` first). With `ffmpeg` on diff --git a/docker-compose.yml b/docker-compose.yml index a24c77c..cf17570 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,6 @@ services: songtrace: - # built here by default; set IMAGE_PREFIX (and IMAGE_TAG) to pull a - # published image instead, see the README's quick start build: . - image: ${IMAGE_PREFIX:-}songtrace:${IMAGE_TAG:-latest} ports: - "8000:8000" volumes: diff --git a/mirror/README.md b/mirror/README.md index 6839a0a..c2dd20f 100644 --- a/mirror/README.md +++ b/mirror/README.md @@ -4,8 +4,7 @@ songtrace is developed on a private forge. `github-mirror.sh` publishes its `main` and the tags on `main` to [github.com/branderson/songtrace](https://github.com/branderson/songtrace), hourly, from a host that can push to GitHub. Issues and pull requests are off -there. Actions stays on, since `.github/workflows/images.yml` builds the image -that `ghcr.io/branderson/songtrace` serves. +there. The mirror's `main` is the forge's plus one commit, which adds a read-only notice under the README's title. Every forge commit is in the mirror unchanged, and @@ -21,8 +20,6 @@ instead of publishing without the notice. pushing it. 3. Once the first sync has pushed, run `github-mirror.sh --settings` (needs `gh`). It turns off issues, pull requests, projects, wiki and discussions. -4. The first image build creates the package as private. In its settings on - GitHub, set its visibility to public. The timer runs a copy of the script, not the checkout's. Reinstall it after reviewing a change to `mirror/`. diff --git a/mirror/github-mirror.sh b/mirror/github-mirror.sh index 2e158bb..a0386c6 100755 --- a/mirror/github-mirror.sh +++ b/mirror/github-mirror.sh @@ -54,9 +54,8 @@ transform() { || die "README.md: no title to put the mirror notice under" } -# Issues, pull requests, wiki and the like off; Actions stays on, since it -# publishes the image. Settings rather than content, so a sync does not -# reapply them; run once after creating the repository. +# Issues, pull requests, wiki and the like off. Settings rather than content, +# so a sync does not reapply them; run once after creating the repository. settings() { command -v gh >/dev/null || die "--settings needs the gh CLI" gh api -X PATCH "repos/$MIRROR_REPO" --silent \ -- 2.54.0