From ff4eff236bbb4e100ef1ac000575f52101200824 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Mon, 7 Sep 2026 07:58:34 -0400 Subject: [PATCH] feat(workflows): cleanup cloudsmith and docker repos --- .github/scripts/cleanup-cloudsmith.mjs | 283 +++++++++++ .github/scripts/cleanup-containers.mjs | 545 +++++++++++++++++++++ .github/workflows/__cleanup-cloudsmith.yml | 54 ++ .github/workflows/__cleanup-containers.yml | 86 ++++ .github/workflows/__cleanup.yml | 41 ++ tests/cleanup-cloudsmith.test.mjs | 72 +++ tests/cleanup-containers.test.mjs | 112 +++++ 7 files changed, 1193 insertions(+) create mode 100644 .github/scripts/cleanup-cloudsmith.mjs create mode 100644 .github/scripts/cleanup-containers.mjs create mode 100644 .github/workflows/__cleanup-cloudsmith.yml create mode 100644 .github/workflows/__cleanup-containers.yml create mode 100644 .github/workflows/__cleanup.yml create mode 100644 tests/cleanup-cloudsmith.test.mjs create mode 100644 tests/cleanup-containers.test.mjs diff --git a/.github/scripts/cleanup-cloudsmith.mjs b/.github/scripts/cleanup-cloudsmith.mjs new file mode 100644 index 00000000..0b551a62 --- /dev/null +++ b/.github/scripts/cleanup-cloudsmith.mjs @@ -0,0 +1,283 @@ +const CLOUDSMITH_API = 'https://api.cloudsmith.io/v1'; +const DEFAULT_GRACE_MS = 24 * 60 * 60 * 1000; + +/** + * Return the package version used by native packages for a GitHub release tag. + * + * @param {string} tag Release tag. + * @returns {string} Native package version prefix. + */ +export function releaseVersion(tag) { + return String(tag ?? '').trim().replace(/^v(?=\d)/i, ''); +} + +/** + * Determine whether a native package version belongs to a GitHub release. + * + * DEB and RPM package versions append their packaging release after a hyphen. + * Exact matches cover formats which do not add a packaging release. + * + * @param {string} packageVersion Cloudsmith package version. + * @param {Iterable} releaseTags Published GitHub release tags. + * @returns {boolean} Whether the package belongs to a published release. + */ +export function packageMatchesRelease(packageVersion, releaseTags) { + const candidate = String(packageVersion ?? '').trim(); + if (!candidate) { + return false; + } + + for (const tag of releaseTags) { + const version = releaseVersion(tag); + if (version && (candidate === version || candidate.startsWith(`${version}-`))) { + return true; + } + } + return false; +} + +/** + * Check all Cloudsmith tag groups for a tag. + * + * @param {object} packageData Cloudsmith package. + * @param {string} tag Package tag. + * @returns {boolean} Whether the tag exists. + */ +export function hasCloudsmithTag(packageData, tag) { + return Object.values(packageData.tags ?? {}).some( + (values) => Array.isArray(values) && values.includes(tag), + ); +} + +/** + * Return published stable and prerelease tags, excluding drafts. + * + * @param {Array} releases GitHub releases. + * @returns {Set} Published release tags. + */ +export function publishedReleaseTags(releases) { + return new Set( + releases + .filter((release) => !release.draft && release.tag_name) + .map((release) => release.tag_name), + ); +} + +function isRecent(timestamp, now, graceMs) { + const time = Date.parse(timestamp ?? ''); + return Number.isFinite(time) && now - time < graceMs; +} + +/** + * Build a fail-closed Cloudsmith cleanup plan. + * + * Package names map case-insensitively to GitHub repository names. Unknown + * package names and packages which Cloudsmith cannot currently delete are kept. + * + * @param {object} options Planning options. + * @param {Array} options.packages Cloudsmith packages. + * @param {Array} options.repositories GitHub repositories. + * @param {Map>} options.releaseTagsByRepository Tags keyed by lower-case repo name. + * @param {number} [options.now] Current time in milliseconds. + * @param {number} [options.graceMs] New-package grace period. + * @returns {{keep: Array, remove: Array}} Cleanup plan. + */ +export function planCloudsmithCleanup({ + packages, + repositories, + releaseTagsByRepository, + now = Date.now(), + graceMs = DEFAULT_GRACE_MS, +}) { + const repositoryNames = new Map( + repositories.map((repository) => [repository.name.toLowerCase(), repository.name]), + ); + const plan = {keep: [], remove: []}; + + for (const packageData of packages) { + const packageName = String(packageData.name ?? ''); + const repositoryName = repositoryNames.get(packageName.toLowerCase()); + let reason; + + if (!repositoryName) { + reason = 'unknown GitHub repository'; + } else if (!packageData.identifier_perm && !packageData.slug_perm) { + reason = 'missing Cloudsmith identifier'; + } else if (packageData.is_deleteable === false) { + reason = 'not deleteable'; + } else if (hasCloudsmithTag(packageData, 'latest')) { + reason = 'latest package for target'; + } else if (isRecent(packageData.uploaded_at, now, graceMs)) { + reason = 'within publish grace period'; + } else { + const tags = releaseTagsByRepository.get(repositoryName.toLowerCase()) ?? new Set(); + if (packageMatchesRelease(packageData.version_orig ?? packageData.version, tags)) { + reason = 'published GitHub release'; + } + } + + const item = {package: packageData, repository: repositoryName, reason}; + if (reason) { + plan.keep.push(item); + } else { + plan.remove.push(item); + } + } + + return plan; +} + +async function responseError(response) { + const body = await response.text(); + return new Error(`${response.status} ${response.statusText}: ${body.slice(0, 500)}`); +} + +async function listCloudsmithPackages({owner, repository, token}) { + const packages = []; + const pageSize = 100; + + for (let page = 1; ; page += 1) { + const url = new URL(`${CLOUDSMITH_API}/packages/${owner}/${repository}/`); + url.searchParams.set('page', page); + url.searchParams.set('page_size', pageSize); + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${token}`, + 'User-Agent': 'LizardByte-artifact-cleanup', + }, + }); + if (!response.ok) { + throw await responseError(response); + } + + const pagePackages = await response.json(); + packages.push(...pagePackages); + const lastPage = Number(response.headers.get('x-pagination-pagetotal')); + if ((Number.isFinite(lastPage) && page >= lastPage) || pagePackages.length < pageSize) { + break; + } + } + + return packages; +} + +async function deleteCloudsmithPackage({owner, repository, token, identifier}) { + const response = await fetch( + `${CLOUDSMITH_API}/packages/${owner}/${repository}/${encodeURIComponent(identifier)}/`, + { + method: 'DELETE', + headers: { + Authorization: `Bearer ${token}`, + 'User-Agent': 'LizardByte-artifact-cleanup', + }, + }, + ); + if (!response.ok && response.status !== 404) { + throw await responseError(response); + } +} + +async function loadRepositories(github, organization) { + return github.paginate(github.rest.repos.listForOrg, { + org: organization, + per_page: 100, + type: 'all', + }); +} + +async function loadReleaseTags(github, organization, repositoryNames) { + const tagsByRepository = new Map(); + for (const repository of repositoryNames) { + const releases = await github.paginate(github.rest.repos.listReleases, { + owner: organization, + repo: repository, + per_page: 100, + }); + tagsByRepository.set(repository.toLowerCase(), publishedReleaseTags(releases)); + } + return tagsByRepository; +} + +function packageLabel(packageData) { + const architecture = (packageData.architectures ?? []).map(({name}) => name).join(','); + const target = [ + packageData.format, + packageData.distro?.slug, + packageData.distro_version?.slug, + architecture, + ].filter(Boolean).join('/'); + return `${packageData.name} ${packageData.version} (${target})`; +} + +/** + * Clean a Cloudsmith repository against the organization's published releases. + * + * @param {object} options Runtime dependencies and settings. + * @returns {Promise} Cleanup counts. + */ +export async function cleanupCloudsmith({ + github, + core, + token, + dryRun, + organization = 'LizardByte', + owner = 'lizardbyte', + repository = 'beta', +}) { + const [packages, repositories] = await Promise.all([ + listCloudsmithPackages({owner, repository, token}), + loadRepositories(github, organization), + ]); + const knownNames = new Set(repositories.map(({name}) => name.toLowerCase())); + const matchedRepositories = new Set( + packages + .map(({name}) => repositories.find( + (repositoryData) => repositoryData.name.toLowerCase() === String(name).toLowerCase(), + )?.name) + .filter(Boolean), + ); + const releaseTagsByRepository = await loadReleaseTags( + github, + organization, + [...matchedRepositories], + ); + const plan = planCloudsmithCleanup({packages, repositories, releaseTagsByRepository}); + + for (const {package: packageData, reason} of plan.keep) { + if (reason === 'unknown GitHub repository') { + core.warning(`Keeping ${packageLabel(packageData)}: ${reason}.`); + } + } + + for (const {package: packageData} of plan.remove) { + const identifier = packageData.identifier_perm ?? packageData.slug_perm; + if (dryRun) { + core.info(`[dry-run] Delete ${packageLabel(packageData)} (${identifier}).`); + } else { + core.info(`Deleting ${packageLabel(packageData)} (${identifier}).`); + await deleteCloudsmithPackage({owner, repository, token, identifier}); + } + } + + const unknownNames = new Set( + packages + .map(({name}) => String(name).toLowerCase()) + .filter((name) => !knownNames.has(name)), + ); + await core.summary + .addHeading(`Cloudsmith ${owner}/${repository} cleanup`, 2) + .addTable([ + [{data: 'Mode', header: true}, {data: dryRun ? 'Dry run' : 'Delete'}], + [{data: 'Packages scanned', header: true}, {data: String(packages.length)}], + [{data: 'Packages retained', header: true}, {data: String(plan.keep.length)}], + [{data: 'Packages selected', header: true}, {data: String(plan.remove.length)}], + [{data: 'Unknown package names', header: true}, {data: String(unknownNames.size)}], + ]) + .write(); + + return { + scanned: packages.length, + retained: plan.keep.length, + selected: plan.remove.length, + }; +} diff --git a/.github/scripts/cleanup-containers.mjs b/.github/scripts/cleanup-containers.mjs new file mode 100644 index 00000000..c60dee05 --- /dev/null +++ b/.github/scripts/cleanup-containers.mjs @@ -0,0 +1,545 @@ +const DOCKER_HUB_API = 'https://hub.docker.com/v2'; +const DOCKER_REGISTRY = 'https://registry-1.docker.io'; +const GHCR_REGISTRY = 'https://ghcr.io'; +const DEFAULT_GRACE_MS = 24 * 60 * 60 * 1000; +const MANIFEST_ACCEPT = [ + 'application/vnd.oci.image.index.v1+json', + 'application/vnd.docker.distribution.manifest.list.v2+json', + 'application/vnd.oci.image.manifest.v1+json', + 'application/vnd.docker.distribution.manifest.v2+json', + 'application/vnd.oci.artifact.manifest.v1+json', +].join(', '); +const MOVING_TAGS = ['latest', 'master', 'test']; + +function isRecent(timestamp, now, graceMs) { + const time = Date.parse(timestamp ?? ''); + return Number.isFinite(time) && now - time < graceMs; +} + +/** + * Return published stable and prerelease tags, excluding drafts. + * + * @param {Array} releases GitHub releases. + * @returns {Set} Published release tags. + */ +export function publishedReleaseTags(releases) { + return new Set( + releases + .filter((release) => !release.draft && release.tag_name) + .map((release) => release.tag_name), + ); +} + +/** + * Check whether a container tag is a retained release or moving alias. + * + * Image variants append a hyphenated suffix to the base tag. + * + * @param {string} tag Container tag. + * @param {Iterable} releaseTags Published GitHub release tags. + * @returns {boolean} Whether the tag must be retained. + */ +export function isRetainedContainerTag(tag, releaseTags) { + const candidate = String(tag ?? ''); + for (const alias of MOVING_TAGS) { + if (candidate === alias || candidate.startsWith(`${alias}-`)) { + return true; + } + } + for (const releaseTag of releaseTags) { + if (candidate === releaseTag || candidate.startsWith(`${releaseTag}-`)) { + return true; + } + } + return false; +} + +/** + * Plan Docker Hub cleanup, collapsing fully stale tag groups to one digest delete. + * + * @param {Array} tags Docker Hub tags. + * @param {Set} releaseTags Published release tags. + * @param {object} [options] Planning options. + * @returns {{actions: Array, retained: number, selectedTags: number}} Cleanup plan. + */ +export function planDockerHubCleanup(tags, releaseTags, { + now = Date.now(), + graceMs = DEFAULT_GRACE_MS, +} = {}) { + const groups = new Map(); + for (const tag of tags) { + const digest = tag.digest || `tag:${tag.name}`; + const item = { + ...tag, + retained: isRetainedContainerTag(tag.name, releaseTags) + || isRecent(tag.last_updated, now, graceMs), + }; + const group = groups.get(digest) ?? []; + group.push(item); + groups.set(digest, group); + } + + const actions = []; + let retained = 0; + let selectedTags = 0; + for (const [digest, group] of groups) { + retained += group.filter(({retained: keep}) => keep).length; + const stale = group.filter(({retained: keep}) => !keep); + selectedTags += stale.length; + if (stale.length === 0) { + continue; + } + + if (!digest.startsWith('tag:') && stale.length === group.length) { + actions.push({ + kind: 'manifest', + digest, + tags: stale.map(({name}) => name), + updatedAt: stale.map(({last_updated}) => last_updated).sort()[0], + }); + } else { + for (const tag of stale) { + actions.push({ + kind: 'tag', + tag: tag.name, + tags: [tag.name], + updatedAt: tag.last_updated, + }); + } + } + } + + actions.sort((left, right) => String(left.updatedAt).localeCompare(String(right.updatedAt))); + return {actions, retained, selectedTags}; +} + +function versionTags(version) { + return version.metadata?.container?.tags ?? []; +} + +/** + * Classify GHCR package versions before resolving retained manifest graphs. + * + * @param {Array} versions GHCR package versions. + * @param {Set} releaseTags Published GitHub release tags. + * @param {object} [options] Planning options. + * @returns {{roots: Array, staleTagged: Array, staleUntagged: Array}} Classification. + */ +export function classifyGhcrVersions(versions, releaseTags, { + now = Date.now(), + graceMs = DEFAULT_GRACE_MS, +} = {}) { + const result = {roots: [], staleTagged: [], staleUntagged: []}; + for (const version of versions) { + const tags = versionTags(version); + const keep = tags.some((tag) => isRetainedContainerTag(tag, releaseTags)) + || isRecent(version.updated_at ?? version.created_at, now, graceMs); + if (keep) { + result.roots.push(version); + } else if (tags.length > 0) { + result.staleTagged.push(version); + } else { + result.staleUntagged.push(version); + } + } + return result; +} + +/** + * Resolve every child manifest reachable from retained GHCR versions. + * + * @param {Iterable} rootDigests Retained manifest digests. + * @param {(digest: string) => Promise} getManifest Manifest loader. + * @returns {Promise>} Reachable manifest digests. + */ +export async function resolveReachableDigests(rootDigests, getManifest) { + const reachable = new Set(); + const pending = [...rootDigests]; + while (pending.length > 0) { + const digest = pending.pop(); + if (!digest || reachable.has(digest)) { + continue; + } + reachable.add(digest); + const manifest = await getManifest(digest); + for (const child of manifest.manifests ?? []) { + if (child.digest && !reachable.has(child.digest)) { + pending.push(child.digest); + } + } + } + return reachable; +} + +/** + * Select stale GHCR versions after the retained manifest graph is known. + * Tagged parent versions are deliberately returned before orphaned children. + * + * @param {object} classification Result from classifyGhcrVersions. + * @param {Set} reachable Digests reachable from retained versions. + * @returns {Array} Versions safe to delete. + */ +export function ghcrCleanupCandidates(classification, reachable) { + return [ + ...classification.staleTagged, + ...classification.staleUntagged.filter(({name}) => !reachable.has(name)), + ]; +} + +async function responseError(response) { + const body = await response.text(); + return new Error(`${response.status} ${response.statusText}: ${body.slice(0, 500)}`); +} + +async function fetchJson(url, options = {}) { + const response = await fetch(url, options); + if (!response.ok) { + throw await responseError(response); + } + return response.json(); +} + +async function dockerHubToken(username, accessToken) { + const response = await fetchJson(`${DOCKER_HUB_API}/auth/token`, { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({identifier: username, secret: accessToken}), + }); + const token = response.access_token ?? response.token; + if (!token) { + throw new Error('Docker Hub authentication did not return an access token.'); + } + return token; +} + +async function paginatedHubResults(url, token) { + const results = []; + let next = url; + while (next) { + const page = await fetchJson(next, { + headers: {Authorization: `Bearer ${token}`}, + }); + results.push(...page.results); + next = page.next; + } + return results; +} + +async function dockerRegistryToken({namespace, repository, username, accessToken}) { + const url = new URL('https://auth.docker.io/token'); + url.searchParams.set('service', 'registry.docker.io'); + url.searchParams.set('scope', `repository:${namespace}/${repository}:pull,push,delete`); + const credentials = Buffer.from(`${username}:${accessToken}`).toString('base64'); + const response = await fetchJson(url, { + headers: {Authorization: `Basic ${credentials}`}, + }); + if (!response.token) { + throw new Error(`Docker Registry authentication failed for ${namespace}/${repository}.`); + } + return response.token; +} + +async function deleteDockerHubTag({namespace, repository, tag, token}) { + const response = await fetch( + `${DOCKER_HUB_API}/namespaces/${encodeURIComponent(namespace)}` + + `/repositories/${encodeURIComponent(repository)}/tags/${encodeURIComponent(tag)}`, + {method: 'DELETE', headers: {Authorization: `Bearer ${token}`}}, + ); + if (!response.ok && response.status !== 404) { + throw await responseError(response); + } +} + +async function deleteDockerManifest({namespace, repository, digest, token}) { + const response = await fetch( + `${DOCKER_REGISTRY}/v2/${namespace}/${repository}/manifests/${digest}`, + {method: 'DELETE', headers: {Authorization: `Bearer ${token}`}}, + ); + if (!response.ok && response.status !== 404) { + throw await responseError(response); + } +} + +async function loadRepositories(github, organization) { + return github.paginate(github.rest.repos.listForOrg, { + org: organization, + per_page: 100, + type: 'all', + }); +} + +async function loadReleaseTags(github, organization, repository) { + const releases = await github.paginate(github.rest.repos.listReleases, { + owner: organization, + repo: repository, + per_page: 100, + }); + return publishedReleaseTags(releases); +} + +function repositoryLookup(repositories) { + return new Map(repositories.map(({name}) => [name.toLowerCase(), name])); +} + +/** + * Clean every Docker Hub repository in a namespace which maps to a GitHub repo. + * + * @param {object} options Runtime dependencies and settings. + * @returns {Promise} Cleanup counts. + */ +export async function cleanupDockerHub({ + github, + core, + username, + accessToken, + dryRun, + maxDeletions, + organization = 'LizardByte', + namespace = 'lizardbyte', +}) { + if (!Number.isInteger(maxDeletions) || maxDeletions < 1) { + throw new Error('maxDeletions must be a positive integer.'); + } + const [hubToken, repositories] = await Promise.all([ + dockerHubToken(username, accessToken), + loadRepositories(github, organization), + ]); + const githubRepositories = repositoryLookup(repositories); + const hubRepositories = await paginatedHubResults( + `${DOCKER_HUB_API}/namespaces/${namespace}/repositories?page_size=100`, + hubToken, + ); + let scanned = 0; + let selectedTags = 0; + let deletedActions = 0; + let remaining = maxDeletions; + let deferredActions = 0; + + for (const hubRepository of hubRepositories.sort((a, b) => a.name.localeCompare(b.name))) { + const githubRepository = githubRepositories.get(hubRepository.name.toLowerCase()); + if (!githubRepository) { + core.warning(`Skipping Docker Hub ${namespace}/${hubRepository.name}: no matching GitHub repository.`); + continue; + } + + const [tags, releaseTags] = await Promise.all([ + paginatedHubResults( + `${DOCKER_HUB_API}/namespaces/${namespace}/repositories/` + + `${encodeURIComponent(hubRepository.name)}/tags?page_size=100`, + hubToken, + ), + loadReleaseTags(github, organization, githubRepository), + ]); + scanned += tags.length; + const plan = planDockerHubCleanup(tags, releaseTags); + selectedTags += plan.selectedTags; + const selectedActions = plan.actions.slice(0, Math.max(remaining, 0)); + deferredActions += plan.actions.length - selectedActions.length; + let registryToken; + + for (const action of selectedActions) { + const description = action.kind === 'manifest' + ? `${namespace}/${hubRepository.name}@${action.digest} (${action.tags.length} tags)` + : `${namespace}/${hubRepository.name}:${action.tag}`; + if (dryRun) { + core.info(`[dry-run] Delete ${description}.`); + } else if (action.kind === 'manifest') { + registryToken ??= await dockerRegistryToken({ + namespace, + repository: hubRepository.name, + username, + accessToken, + }); + core.info(`Deleting ${description}.`); + await deleteDockerManifest({ + namespace, + repository: hubRepository.name, + digest: action.digest, + token: registryToken, + }); + } else { + core.info(`Deleting ${description}.`); + await deleteDockerHubTag({ + namespace, + repository: hubRepository.name, + tag: action.tag, + token: hubToken, + }); + } + deletedActions += 1; + remaining -= 1; + } + } + + await core.summary + .addHeading(`Docker Hub ${namespace} cleanup`, 2) + .addTable([ + [{data: 'Mode', header: true}, {data: dryRun ? 'Dry run' : 'Delete'}], + [{data: 'Repositories scanned', header: true}, {data: String(hubRepositories.length)}], + [{data: 'Tags scanned', header: true}, {data: String(scanned)}], + [{data: 'Tags selected', header: true}, {data: String(selectedTags)}], + [{data: 'Delete operations selected', header: true}, {data: String(deletedActions)}], + [{data: 'Operations deferred', header: true}, {data: String(deferredActions)}], + ]) + .write(); + + return {repositories: hubRepositories.length, scanned, selectedTags, deletedActions, deferredActions}; +} + +async function ghcrToken({namespace, packageName, username, accessToken}) { + const url = new URL(`${GHCR_REGISTRY}/token`); + url.searchParams.set('service', 'ghcr.io'); + url.searchParams.set('scope', `repository:${namespace}/${packageName}:pull`); + const credentials = Buffer.from(`${username}:${accessToken}`).toString('base64'); + const response = await fetchJson(url, { + headers: {Authorization: `Basic ${credentials}`}, + }); + const token = response.token ?? response.access_token; + if (!token) { + throw new Error(`GHCR authentication failed for ${namespace}/${packageName}.`); + } + return token; +} + +async function getGhcrManifest({namespace, packageName, digest, token}) { + const packagePath = packageName.split('/').map(encodeURIComponent).join('/'); + return fetchJson(`${GHCR_REGISTRY}/v2/${namespace}/${packagePath}/manifests/${digest}`, { + headers: { + Accept: MANIFEST_ACCEPT, + Authorization: `Bearer ${token}`, + }, + }); +} + +async function deleteGhcrVersion({github, organization, packageName, versionId}) { + await github.request( + 'DELETE /orgs/{org}/packages/{package_type}/{package_name}/versions/{package_version_id}', + { + org: organization, + package_type: 'container', + package_name: packageName, + package_version_id: versionId, + }, + ); +} + +/** + * Clean GHCR package versions while preserving retained multi-arch graphs. + * + * @param {object} options Runtime dependencies and settings. + * @returns {Promise} Cleanup counts. + */ +export async function cleanupGhcr({ + github, + core, + username, + accessToken, + dryRun, + maxDeletions, + organization = 'LizardByte', + namespace = 'lizardbyte', +}) { + if (!Number.isInteger(maxDeletions) || maxDeletions < 1) { + throw new Error('maxDeletions must be a positive integer.'); + } + const [packages, repositories] = await Promise.all([ + github.paginate('GET /orgs/{org}/packages', { + org: organization, + package_type: 'container', + per_page: 100, + }), + loadRepositories(github, organization), + ]); + const githubRepositories = repositoryLookup(repositories); + let scanned = 0; + let selected = 0; + let operations = 0; + let deferred = 0; + let remaining = maxDeletions; + + for (const packageData of packages.sort((a, b) => a.name.localeCompare(b.name))) { + const githubRepository = githubRepositories.get(packageData.name.toLowerCase()); + if (!githubRepository) { + core.warning(`Skipping GHCR ${namespace}/${packageData.name}: no matching GitHub repository.`); + continue; + } + + const [versions, releaseTags] = await Promise.all([ + github.paginate( + 'GET /orgs/{org}/packages/{package_type}/{package_name}/versions', + { + org: organization, + package_type: 'container', + package_name: packageData.name, + per_page: 100, + }, + ), + loadReleaseTags(github, organization, githubRepository), + ]); + scanned += versions.length; + const classification = classifyGhcrVersions(versions, releaseTags); + let reachable; + try { + const registryToken = await ghcrToken({ + namespace, + packageName: packageData.name, + username, + accessToken, + }); + reachable = await resolveReachableDigests( + classification.roots.map(({name}) => name), + (digest) => getGhcrManifest({ + namespace, + packageName: packageData.name, + digest, + token: registryToken, + }), + ); + } catch (error) { + core.warning( + `Skipping GHCR ${namespace}/${packageData.name}: retained manifest graph could not be read: ` + + error.message, + ); + continue; + } + + const candidates = ghcrCleanupCandidates(classification, reachable); + selected += candidates.length; + const selectedVersions = candidates.slice(0, Math.max(remaining, 0)); + deferred += candidates.length - selectedVersions.length; + + for (const version of selectedVersions) { + const tags = versionTags(version); + const description = tags.length > 0 + ? `${namespace}/${packageData.name}:${tags.join(',')}` + : `${namespace}/${packageData.name}@${version.name}`; + if (dryRun) { + core.info(`[dry-run] Delete ${description} (version ${version.id}).`); + } else { + core.info(`Deleting ${description} (version ${version.id}).`); + await deleteGhcrVersion({ + github, + organization, + packageName: packageData.name, + versionId: version.id, + }); + } + operations += 1; + remaining -= 1; + } + } + + await core.summary + .addHeading(`GHCR ${namespace} cleanup`, 2) + .addTable([ + [{data: 'Mode', header: true}, {data: dryRun ? 'Dry run' : 'Delete'}], + [{data: 'Packages scanned', header: true}, {data: String(packages.length)}], + [{data: 'Versions scanned', header: true}, {data: String(scanned)}], + [{data: 'Versions selected', header: true}, {data: String(selected)}], + [{data: 'Delete operations selected', header: true}, {data: String(operations)}], + [{data: 'Versions deferred', header: true}, {data: String(deferred)}], + ]) + .write(); + + return {packages: packages.length, scanned, selected, operations, deferred}; +} diff --git a/.github/workflows/__cleanup-cloudsmith.yml b/.github/workflows/__cleanup-cloudsmith.yml new file mode 100644 index 00000000..503a3711 --- /dev/null +++ b/.github/workflows/__cleanup-cloudsmith.yml @@ -0,0 +1,54 @@ +--- +# Reusable cleanup for packages in the lizardbyte/beta Cloudsmith repository. + +name: Cleanup Cloudsmith (called) +permissions: {} + +on: + workflow_call: + inputs: + dry_run: + description: Report packages without deleting them + required: false + type: boolean + default: true + secrets: + CLOUDSMITH_API_KEY: + description: Cloudsmith token with delete access to lizardbyte/beta + required: true + GH_BOT_TOKEN: + description: GitHub token used to enumerate repositories and releases + required: true + +jobs: + cleanup: + name: Cloudsmith beta + if: github.repository == 'LizardByte/.github' + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Clean packages + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }} + DRY_RUN: ${{ inputs.dry_run }} + with: + github-token: ${{ secrets.GH_BOT_TOKEN }} + script: | + const path = require('node:path'); + const {pathToFileURL} = require('node:url'); + const modulePath = path.join( + process.env.GITHUB_WORKSPACE, + '.github/scripts/cleanup-cloudsmith.mjs', + ); + const {cleanupCloudsmith} = await import(pathToFileURL(modulePath).href); + await cleanupCloudsmith({ + github, + core, + token: process.env.CLOUDSMITH_API_KEY, + dryRun: process.env.DRY_RUN === 'true', + }); diff --git a/.github/workflows/__cleanup-containers.yml b/.github/workflows/__cleanup-containers.yml new file mode 100644 index 00000000..50322e36 --- /dev/null +++ b/.github/workflows/__cleanup-containers.yml @@ -0,0 +1,86 @@ +--- +# Reusable cleanup for every Docker Hub and GHCR container under lizardbyte. + +name: Cleanup containers (called) +permissions: {} + +on: + workflow_call: + inputs: + docker_hub_username: + description: Docker Hub username associated with the access token + required: true + type: string + dry_run: + description: Report container versions without deleting them + required: false + type: boolean + default: true + gh_bot_name: + description: GitHub username associated with the package token + required: true + type: string + max_deletions: + description: Maximum delete operations per registry and run + required: false + type: number + default: 1000 + secrets: + DOCKER_HUB_ACCESS_TOKEN: + description: Docker Hub token with read, write, and delete access + required: true + GH_BOT_TOKEN: + description: GitHub token with read:packages and delete:packages scopes + required: true + +jobs: + cleanup: + name: ${{ matrix.name }} + if: github.repository == 'LizardByte/.github' + permissions: + contents: read + packages: write + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: Docker Hub + registry: docker-hub + - name: GitHub Container Registry + registry: ghcr + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Clean images + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + DRY_RUN: ${{ inputs.dry_run }} + MAX_DELETIONS: ${{ inputs.max_deletions }} + REGISTRY: ${{ matrix.registry }} + REGISTRY_ACCESS_TOKEN: >- + ${{ matrix.registry == 'docker-hub' && secrets.DOCKER_HUB_ACCESS_TOKEN || secrets.GH_BOT_TOKEN }} + REGISTRY_USERNAME: >- + ${{ matrix.registry == 'docker-hub' && inputs.docker_hub_username || inputs.gh_bot_name }} + with: + github-token: ${{ secrets.GH_BOT_TOKEN }} + script: | + const path = require('node:path'); + const {pathToFileURL} = require('node:url'); + const modulePath = path.join( + process.env.GITHUB_WORKSPACE, + '.github/scripts/cleanup-containers.mjs', + ); + const {cleanupDockerHub, cleanupGhcr} = await import(pathToFileURL(modulePath).href); + const cleanup = process.env.REGISTRY === 'docker-hub' + ? cleanupDockerHub + : cleanupGhcr; + await cleanup({ + github, + core, + username: process.env.REGISTRY_USERNAME, + accessToken: process.env.REGISTRY_ACCESS_TOKEN, + dryRun: process.env.DRY_RUN === 'true', + maxDeletions: Number(process.env.MAX_DELETIONS), + }); diff --git a/.github/workflows/__cleanup.yml b/.github/workflows/__cleanup.yml new file mode 100644 index 00000000..220374f0 --- /dev/null +++ b/.github/workflows/__cleanup.yml @@ -0,0 +1,41 @@ +--- +# Clean LizardByte package and container registries against published GitHub releases. + +name: Cleanup artifacts +permissions: {} + +on: + workflow_dispatch: + +concurrency: + group: cleanup-artifacts + cancel-in-progress: false + +jobs: + cloudsmith: + name: Cloudsmith + uses: ./.github/workflows/__cleanup-cloudsmith.yml + if: github.repository == 'LizardByte/.github' + permissions: + contents: read + with: + dry_run: true + secrets: + CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_API_KEY }} + GH_BOT_TOKEN: ${{ secrets.GH_BOT_TOKEN }} + + containers: + name: Containers + uses: ./.github/workflows/__cleanup-containers.yml + if: github.repository == 'LizardByte/.github' + permissions: + contents: read + packages: write + with: + docker_hub_username: ${{ vars.DOCKER_HUB_USERNAME }} + dry_run: true + gh_bot_name: ${{ vars.GH_BOT_NAME }} + max_deletions: 1000 + secrets: + DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + GH_BOT_TOKEN: ${{ secrets.GH_BOT_TOKEN }} diff --git a/tests/cleanup-cloudsmith.test.mjs b/tests/cleanup-cloudsmith.test.mjs new file mode 100644 index 00000000..81f6ea3a --- /dev/null +++ b/tests/cleanup-cloudsmith.test.mjs @@ -0,0 +1,72 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + hasCloudsmithTag, + packageMatchesRelease, + planCloudsmithCleanup, + publishedReleaseTags, + releaseVersion, +} from '../.github/scripts/cleanup-cloudsmith.mjs'; + +test('releaseVersion removes only a numeric v prefix', () => { + assert.equal(releaseVersion('v2026.906.123708'), '2026.906.123708'); + assert.equal(releaseVersion('0.5.0'), '0.5.0'); + assert.equal(releaseVersion('version-one'), 'version-one'); +}); + +test('packageMatchesRelease supports exact and native package release suffixes', () => { + const tags = new Set(['v2026.516.143833', 'v0.23.1']); + assert.equal(packageMatchesRelease('2026.516.143833-1+ubuntu24.04', tags), true); + assert.equal(packageMatchesRelease('2026.516.143833-1.fc44', tags), true); + assert.equal(packageMatchesRelease('0.23.1', tags), true); + assert.equal(packageMatchesRelease('0.23.10-1.fc44', tags), false); + assert.equal(packageMatchesRelease('2026.906.123708-1.fc44', tags), false); +}); + +test('hasCloudsmithTag finds automatic tags in any tag group', () => { + assert.equal(hasCloudsmithTag({tags: {version: ['latest']}}, 'latest'), true); + assert.equal(hasCloudsmithTag({tags: {custom: ['latest']}}, 'latest'), true); + assert.equal(hasCloudsmithTag({tags: {}}, 'latest'), false); +}); + +test('publishedReleaseTags keeps stable and prereleases but excludes drafts', () => { + const tags = publishedReleaseTags([ + {tag_name: 'v1.0.0', draft: false, prerelease: false}, + {tag_name: 'v1.1.0-beta', draft: false, prerelease: true}, + {tag_name: 'v2.0.0', draft: true, prerelease: false}, + ]); + assert.deepEqual([...tags], ['v1.0.0', 'v1.1.0-beta']); +}); + +test('planCloudsmithCleanup retains releases, latest targets, recent and unknown packages', () => { + const now = Date.parse('2026-09-06T12:00:00Z'); + const packageData = (overrides) => ({ + name: 'sunshine', + version: '2026.900.000000-1.fc44', + uploaded_at: '2026-09-01T00:00:00Z', + identifier_perm: 'package', + tags: {}, + ...overrides, + }); + const packages = [ + packageData({identifier_perm: 'release', version: '2026.516.143833-1.fc44'}), + packageData({identifier_perm: 'latest', tags: {version: ['latest']}}), + packageData({identifier_perm: 'recent', uploaded_at: '2026-09-06T11:00:00Z'}), + packageData({identifier_perm: 'unknown', name: 'future-package'}), + packageData({identifier_perm: 'stale'}), + ]; + const plan = planCloudsmithCleanup({ + packages, + repositories: [{name: 'Sunshine'}], + releaseTagsByRepository: new Map([ + ['sunshine', new Set(['v2026.516.143833'])], + ]), + now, + }); + + assert.deepEqual(plan.keep.map(({package: item}) => item.identifier_perm), [ + 'release', 'latest', 'recent', 'unknown', + ]); + assert.deepEqual(plan.remove.map(({package: item}) => item.identifier_perm), ['stale']); +}); diff --git a/tests/cleanup-containers.test.mjs b/tests/cleanup-containers.test.mjs new file mode 100644 index 00000000..327db178 --- /dev/null +++ b/tests/cleanup-containers.test.mjs @@ -0,0 +1,112 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + classifyGhcrVersions, + ghcrCleanupCandidates, + isRetainedContainerTag, + planDockerHubCleanup, + publishedReleaseTags, + resolveReachableDigests, +} from '../.github/scripts/cleanup-containers.mjs'; + +test('publishedReleaseTags includes stable and prerelease releases but not drafts', () => { + assert.deepEqual( + [...publishedReleaseTags([ + {tag_name: 'v1.0.0', draft: false, prerelease: false}, + {tag_name: 'v1.1.0', draft: false, prerelease: true}, + {tag_name: 'v2.0.0', draft: true, prerelease: false}, + ])], + ['v1.0.0', 'v1.1.0'], + ); +}); + +test('isRetainedContainerTag supports release variants and moving aliases', () => { + const releases = new Set(['v2026.516.143833']); + assert.equal(isRetainedContainerTag('v2026.516.143833', releases), true); + assert.equal(isRetainedContainerTag('v2026.516.143833-ubuntu-24.04', releases), true); + assert.equal(isRetainedContainerTag('latest-clion-toolchain', releases), true); + assert.equal(isRetainedContainerTag('master', releases), true); + assert.equal(isRetainedContainerTag('test-debian', releases), true); + assert.equal(isRetainedContainerTag('latestish', releases), false); + assert.equal(isRetainedContainerTag('v2026.516.1438330', releases), false); + assert.equal(isRetainedContainerTag('abcdef0-ubuntu-24.04', releases), false); +}); + +test('planDockerHubCleanup groups fully stale tags by digest', () => { + const now = Date.parse('2026-09-06T12:00:00Z'); + const old = '2026-09-01T00:00:00Z'; + const recent = '2026-09-06T11:00:00Z'; + const plan = planDockerHubCleanup([ + {name: 'v1.0.0', digest: 'sha256:keep', last_updated: old}, + {name: 'abcdef0', digest: 'sha256:keep', last_updated: old}, + {name: 'old-release', digest: 'sha256:stale', last_updated: old}, + {name: '1234567', digest: 'sha256:stale', last_updated: old}, + {name: 'new-build', digest: 'sha256:recent', last_updated: recent}, + ], new Set(['v1.0.0']), {now}); + + assert.equal(plan.retained, 2); + assert.equal(plan.selectedTags, 3); + assert.deepEqual( + plan.actions.map(({kind, digest, tag}) => [kind, digest, tag]), + [ + ['tag', undefined, 'abcdef0'], + ['manifest', 'sha256:stale', undefined], + ], + ); +}); + +test('classifyGhcrVersions protects retained tags and the publish grace period', () => { + const now = Date.parse('2026-09-06T12:00:00Z'); + const version = (id, tags, updatedAt) => ({ + id, + name: `sha256:${id}`, + updated_at: updatedAt, + metadata: {container: {tags}}, + }); + const classification = classifyGhcrVersions([ + version('release', ['v1.0.0', 'abcdef0'], '2026-09-01T00:00:00Z'), + version('recent', [], '2026-09-06T11:00:00Z'), + version('tagged', ['old'], '2026-09-01T00:00:00Z'), + version('untagged', [], '2026-09-01T00:00:00Z'), + ], new Set(['v1.0.0']), {now}); + + assert.deepEqual(classification.roots.map(({id}) => id), ['release', 'recent']); + assert.deepEqual(classification.staleTagged.map(({id}) => id), ['tagged']); + assert.deepEqual(classification.staleUntagged.map(({id}) => id), ['untagged']); +}); + +test('resolveReachableDigests walks nested multi-arch manifest indexes', async () => { + const manifests = new Map([ + ['sha256:root', {manifests: [{digest: 'sha256:amd64'}, {digest: 'sha256:arm64'}]}], + ['sha256:amd64', {manifests: [{digest: 'sha256:provenance'}]}], + ['sha256:arm64', {}], + ['sha256:provenance', {}], + ]); + const reachable = await resolveReachableDigests( + ['sha256:root'], + async (digest) => manifests.get(digest), + ); + + assert.deepEqual( + [...reachable].sort(), + ['sha256:amd64', 'sha256:arm64', 'sha256:provenance', 'sha256:root'], + ); +}); + +test('ghcrCleanupCandidates preserves reachable untagged child manifests', () => { + const classification = { + roots: [{id: 'root', name: 'sha256:root'}], + staleTagged: [{id: 'tagged', name: 'sha256:tagged'}], + staleUntagged: [ + {id: 'child', name: 'sha256:child'}, + {id: 'orphan', name: 'sha256:orphan'}, + ], + }; + const candidates = ghcrCleanupCandidates( + classification, + new Set(['sha256:root', 'sha256:child']), + ); + + assert.deepEqual(candidates.map(({id}) => id), ['tagged', 'orphan']); +});