From 0644ec8f5a147b99559c84f5216093ce95325731 Mon Sep 17 00:00:00 2001 From: lilleman Date: Sat, 4 Jul 2026 09:18:07 +0200 Subject: [PATCH] CI: nightly registry cleanup - prune hash images not release-tagged nor a branch head --- .gitea/workflows/registry-cleanup.yml | 22 ++++++ README.md | 16 ++-- package.json | 2 +- registry-cleanup/cleanup.ts | 98 ++++++++++++++++++++++++ registry-cleanup/select-versions.test.ts | 48 ++++++++++++ registry-cleanup/select-versions.ts | 31 ++++++++ tsconfig.json | 2 +- 7 files changed, 212 insertions(+), 7 deletions(-) create mode 100644 .gitea/workflows/registry-cleanup.yml create mode 100644 registry-cleanup/cleanup.ts create mode 100644 registry-cleanup/select-versions.test.ts create mode 100644 registry-cleanup/select-versions.ts diff --git a/.gitea/workflows/registry-cleanup.yml b/.gitea/workflows/registry-cleanup.yml new file mode 100644 index 0000000..335f0c0 --- /dev/null +++ b/.gitea/workflows/registry-cleanup.yml @@ -0,0 +1,22 @@ +name: Registry cleanup +on: + schedule: + - cron: '43 3 * * *' + workflow_dispatch: + +jobs: + prune-stale-images: + runs-on: docker-host + steps: + - uses: actions/checkout@v4.2.2 + - name: Delete hash images that are neither release-tagged nor a branch head + env: + REGISTRY_TOKEN: ${{ secrets.DOCKER_REGISTRY_TOKEN }} + REGISTRY_USER: ${{ vars.DOCKER_REGISTRY_USER }} + REPO_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + docker run --rm -v "$PWD:/repo" -w /repo \ + -e REGISTRY_TOKEN -e REGISTRY_USER -e REPO_TOKEN -e REPOSITORY -e SERVER_URL \ + node:24.16.0-alpine3.24 node registry-cleanup/cleanup.ts diff --git a/README.md b/README.md index f6a6ac1..04a803b 100644 --- a/README.md +++ b/README.md @@ -1176,6 +1176,7 @@ Gitea Actions (`.gitea/workflows/`) runs the pipeline; the test job runs | `ci.yml` | push, any branch except `main` | the full gate (`bash ci.sh`), then build + push the app image | | `release.yml` | push of a `vX.Y.Z` tag | re-tag that commit's image as `X.Y.Z`, `X.Y`, `X`, `latest` | | `mirror.yml` | push to `main` or any tag, or manual | force-push `main` + tags to the [GitHub mirror](https://github.com/larvit/plainpages) | +| `registry-cleanup.yml` | nightly cron, or manual | delete registry images that are neither release-tagged nor a branch head | `main` is not re-tested on push — its commits are meant to arrive already green from a gated branch, so the status check to gate a merge on is `CI / full-gate (push)`. @@ -1197,11 +1198,16 @@ as the Actions **variable** `DOCKER_REGISTRY_USER` and the token as the Actions this step runs inside the required gate, a missing/expired token (or registry outage) fails every branch's gate and blocks **all** merges until restored — set the secrets before this lands, and use a -non-expiring token or track its expiry. Retention: hash tags -accumulate one image per gated push, so an org-level package **cleanup rule** -(org Settings → Packages, applied by Gitea's daily cleanup cron) prunes them — type -`container`, remove versions matching `^[0-9a-f]{40}$` older than 30 days, keep the 10 most -recent and anything matching `^\d+(\.\d+){0,2}$|^latest$` (release tags are never pruned). +non-expiring token or track its expiry. Retention: hash tags accumulate one image per gated +push, so the nightly `registry-cleanup.yml` prunes them precisely +([`registry-cleanup/cleanup.ts`](registry-cleanup/cleanup.ts), run in a `node:24` container): +a hash tag survives only while its commit is a **branch head** or carries a **`vX.Y.Z` +release tag**; deleted alongside are the untagged `sha256:…` child manifests (arch image + +provenance) that no surviving tag references. Named tags (`1.2.3`, `latest`, …) are never +touched. It reuses `DOCKER_REGISTRY_USER`/`DOCKER_REGISTRY_TOKEN` — no extra setup. Don't +add a pattern-based org cleanup rule for this package (and remove it if one exists): its +age/count heuristics can't see branch heads or release tags and would delete images the +workflow protects. **Releases** — pushing a semver git tag (`git tag v1.2.3 && git push origin v1.2.3`) runs `release.yml`, which pulls that commit's hash image from the registry and re-tags it as diff --git a/package.json b/package.json index 5128a9d..b822a4e 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "dev": "node --watch src/server.ts", "gen-jwks": "node src/auth/gen-jwks.ts", "typecheck": "tsc --noEmit", - "test": "node --test \"src/**/*.test.ts\" \"plugins/**/*.test.ts\" \"examples/**/*.test.ts\"" + "test": "node --test \"src/**/*.test.ts\" \"plugins/**/*.test.ts\" \"examples/**/*.test.ts\" \"registry-cleanup/**/*.test.ts\"" }, "dependencies": { "@larvit/log": "2.3.0", diff --git a/registry-cleanup/cleanup.ts b/registry-cleanup/cleanup.ts new file mode 100644 index 0000000..ed2a0b1 --- /dev/null +++ b/registry-cleanup/cleanup.ts @@ -0,0 +1,98 @@ +// Prunes the app's container package in the Gitea registry: a commit-hash tag survives only +// while its commit is a branch head or carries a vX.Y.Z release tag. Untagged sha256:* package +// versions are CHILD manifests (arch image + provenance) of the tagged OCI indexes — deleting +// one that a surviving tag still references breaks that image, so only the ones no kept tag +// references are removed. Named tags (1.2.3, latest, …) are never touched. +import { planHashTagDeletions, selectOrphanedManifests } from "./select-versions.ts"; + +function env(name: string): string { + const value = process.env[name]; + if (value === undefined || value === "") throw new Error(`Missing env var ${name}`); + return value; +} + +const registryToken = env("REGISTRY_TOKEN"); +const registryUser = env("REGISTRY_USER"); +const repoToken = env("REPO_TOKEN"); +const repository = env("REPOSITORY"); +const serverUrl = env("SERVER_URL").replace(/\/+$/, ""); + +const [owner, name] = repository.split("/"); +if (owner === undefined || name === undefined || owner === "" || name === "") { + throw new Error(`REPOSITORY must be owner/name, got "${repository}"`); +} + +async function fetchOk(url: string, init: RequestInit): Promise { + const res = await fetch(url, init); + if (!res.ok) throw new Error(`${init.method ?? "GET"} ${url} -> ${res.status}: ${await res.text()}`); + return res; +} + +async function apiGetAllPages(path: string, token: string): Promise { + const all: T[] = []; + const limit = 50; + for (let page = 1; ; page++) { + const sep = path.includes("?") ? "&" : "?"; + const res = await fetchOk(`${serverUrl}/api/v1${path}${sep}limit=${limit}&page=${page}`, { + headers: { authorization: `token ${token}` }, + }); + const batch = (await res.json()) as T[]; + // Stop on an empty page, not a short one — a lowered server page-size cap would otherwise + // silently truncate the list, and a truncated branch/tag list deletes protected images. + if (batch.length === 0) return all; + all.push(...batch); + } +} + +async function childDigests(tag: string): Promise { + const res = await fetchOk(`${serverUrl}/v2/${owner}/${name}/manifests/${tag}`, { + headers: { + accept: [ + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", + ].join(", "), + authorization: `Basic ${Buffer.from(`${registryUser}:${registryToken}`).toString("base64")}`, + }, + }); + const manifest = (await res.json()) as { manifests?: { digest: string }[] }; + return (manifest.manifests ?? []).map((m) => m.digest); +} + +async function deleteVersion(version: string): Promise { + const url = `${serverUrl}/api/v1/packages/${owner}/container/${name}/${encodeURIComponent(version)}`; + const res = await fetch(url, { headers: { authorization: `token ${registryToken}` }, method: "DELETE" }); + if (res.status === 404) { + console.log(`already gone: ${version}`); + return; + } + if (!res.ok) throw new Error(`DELETE ${url} -> ${res.status}: ${await res.text()}`); + console.log(`deleted: ${version}`); +} + +const branches = await apiGetAllPages<{ commit: { id: string } }>(`/repos/${owner}/${name}/branches`, repoToken); +if (branches.length === 0) throw new Error("no branches returned — refusing to prune with an empty keep-set"); +const tags = await apiGetAllPages<{ commit: { sha: string }; name: string }>(`/repos/${owner}/${name}/tags`, repoToken); +const packages = await apiGetAllPages<{ name: string; version: string }>( + `/packages/${owner}?type=container&q=${encodeURIComponent(name)}`, + registryToken, +); + +const versions = packages.filter((p) => p.name === name).map((p) => p.version); +const plan = planHashTagDeletions({ + branchHeadShas: branches.map((b) => b.commit.id), + releaseTagShas: tags.filter((t) => /^v\d+\.\d+\.\d+$/.test(t.name)).map((t) => t.commit.sha), + versions, +}); + +const referenced = new Set(); +for (const tag of plan.keptTags) { + for (const digest of await childDigests(tag)) referenced.add(digest); +} +const orphans = selectOrphanedManifests({ referencedDigests: [...referenced], versions }); + +for (const version of [...plan.deletions, ...orphans]) await deleteVersion(version); +console.log( + `kept ${plan.keptTags.length} tags; deleted ${plan.deletions.length} hash tags and ${orphans.length} orphaned manifests`, +); diff --git a/registry-cleanup/select-versions.test.ts b/registry-cleanup/select-versions.test.ts new file mode 100644 index 0000000..66a1d2b --- /dev/null +++ b/registry-cleanup/select-versions.test.ts @@ -0,0 +1,48 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { planHashTagDeletions, selectOrphanedManifests } from "./select-versions.ts"; + +const headSha = "0582809000000000000000000000000000000001"; +const releaseSha = "50006dd000000000000000000000000000000002"; +const staleSha = "c8981c1000000000000000000000000000000003"; + +test("planHashTagDeletions deletes hash tags that are neither a branch head nor release-tagged", () => { + const plan = planHashTagDeletions({ + branchHeadShas: [headSha], + releaseTagShas: [releaseSha], + versions: [headSha, releaseSha, staleSha], + }); + assert.deepEqual(plan.deletions, [staleSha]); + assert.deepEqual(plan.keptTags, [headSha, releaseSha]); +}); + +test("planHashTagDeletions keeps named tags and excludes sha256 manifest versions from keptTags", () => { + const plan = planHashTagDeletions({ + branchHeadShas: [], + releaseTagShas: [], + versions: ["0.0.1", "0", "latest", "some-manual-tag", `sha256:${"a".repeat(64)}`, staleSha], + }); + assert.deepEqual(plan.deletions, [staleSha]); + assert.deepEqual(plan.keptTags, ["0.0.1", "0", "latest", "some-manual-tag"]); +}); + +test("planHashTagDeletions with no versions plans nothing", () => { + const plan = planHashTagDeletions({ branchHeadShas: [headSha], releaseTagShas: [], versions: [] }); + assert.deepEqual(plan.deletions, []); + assert.deepEqual(plan.keptTags, []); +}); + +test("selectOrphanedManifests picks only sha256 versions unreferenced by kept tags", () => { + const referenced = `sha256:${"b".repeat(64)}`; + const orphaned = `sha256:${"c".repeat(64)}`; + const orphans = selectOrphanedManifests({ + referencedDigests: [referenced], + versions: ["0.0.1", "latest", headSha, referenced, orphaned], + }); + assert.deepEqual(orphans, [orphaned]); +}); + +test("selectOrphanedManifests with nothing referenced orphans every sha256 version", () => { + const manifest = `sha256:${"d".repeat(64)}`; + assert.deepEqual(selectOrphanedManifests({ referencedDigests: [], versions: [manifest, "latest"] }), [manifest]); +}); diff --git a/registry-cleanup/select-versions.ts b/registry-cleanup/select-versions.ts new file mode 100644 index 0000000..d9bb40c --- /dev/null +++ b/registry-cleanup/select-versions.ts @@ -0,0 +1,31 @@ +const hashTag = /^[0-9a-f]{40}$/; +const manifestVersion = /^sha256:[0-9a-f]{64}$/; + +export type HashTagPlan = { + deletions: string[]; + keptTags: string[]; +}; + +export function planHashTagDeletions(input: { + branchHeadShas: string[]; + releaseTagShas: string[]; + versions: string[]; +}): HashTagPlan { + const keep = new Set([...input.branchHeadShas, ...input.releaseTagShas]); + const deletions: string[] = []; + const keptTags: string[] = []; + for (const version of input.versions) { + if (manifestVersion.test(version)) continue; + if (hashTag.test(version) && !keep.has(version)) deletions.push(version); + else keptTags.push(version); + } + return { deletions, keptTags }; +} + +export function selectOrphanedManifests(input: { + referencedDigests: string[]; + versions: string[]; +}): string[] { + const referenced = new Set(input.referencedDigests); + return input.versions.filter((v) => manifestVersion.test(v) && !referenced.has(v)); +} diff --git a/tsconfig.json b/tsconfig.json index 4dbac13..a6d559f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -24,5 +24,5 @@ "forceConsistentCasingInFileNames": true, "skipLibCheck": true }, - "include": ["config", "examples/config", "examples/plugins", "plugins", "src"] + "include": ["config", "examples/config", "examples/plugins", "plugins", "registry-cleanup", "src"] }