diff --git a/.changeset/native-update-release-assets.md b/.changeset/native-update-release-assets.md new file mode 100644 index 000000000..ccd141171 --- /dev/null +++ b/.changeset/native-update-release-assets.md @@ -0,0 +1,5 @@ +--- +'@pymodel/pythinker-code': patch +--- + +Native `pythinker update` downloads the new binary from the GitHub release again; native installs on 2.2.0–2.4.1 need one reinstall to receive it (see #354). diff --git a/README.md b/README.md index 5a4ec66a0..e586616a1 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,8 @@ The CLI ships as a native binary, so there is no Node.js prerequisite. | Nix | `nix run github:PyModel/pythinker-code` | | npm | `npm install -g @pymodel/pythinker-code` (needs Node.js 24.15+) | +> Native install on 2.2.0–2.4.1? `pythinker update` cannot download new versions there. Run the install command again once to get a later version ([#354](https://github.com/PyModel/pythinker-code/issues/354)). + ```sh cd your-project pythinker diff --git a/apps/pythinker-code/scripts/native/produce-manifest.mjs b/apps/pythinker-code/scripts/native/produce-manifest.mjs index 948c350a9..0e7b1bdd0 100644 --- a/apps/pythinker-code/scripts/native/produce-manifest.mjs +++ b/apps/pythinker-code/scripts/native/produce-manifest.mjs @@ -9,6 +9,10 @@ * (produced by package.mjs across the 6 native-build matrix runners). The * zip is the only form in which binaries leave the matrix runners, so this * script extracts each bare executable and emits next to it: + * pythinker-code-[.exe] the bare binary; `filename` in the + * manifest and `url` in the CDN latest.json + * both name it, and updaters without zstd + * support download it * pythinker-code-.zst zstd -19, consumed by the staged updater * pythinker-code-.tar.gz consumed by install.sh / install.ps1 * .sha256 sidecars in ` ` format @@ -23,7 +27,7 @@ import { execFile } from 'node:child_process'; import { createHash } from 'node:crypto'; import { createReadStream } from 'node:fs'; -import { mkdtemp, readdir, rm, writeFile } from 'node:fs/promises'; +import { copyFile, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { basename, join, resolve } from 'node:path'; import { promisify } from 'node:util'; @@ -71,8 +75,8 @@ for (const sumFile of sumFiles.sort()) { const target = basename(sumFile, '.sha256').replace(/^pythinker-code-/, '').replace(/\.zip$/, ''); const zipName = `pythinker-code-${target}.zip`; const exeName = target.startsWith('win32') ? 'pythinker.exe' : 'pythinker'; - // The CDN bare-binary layout carries the .exe suffix on Windows - // (src/constant/app.ts); the updater's fallback downloads this filename. + // Windows keeps the .exe suffix. Every file the manifest names is uploaded + // to the release, so the updater can always fetch it. const binaryName = target.startsWith('win32') ? `pythinker-code-${target}.exe` : `pythinker-code-${target}`; const artifactBase = `pythinker-code-${target}`; const zstName = `${artifactBase}.zst`; @@ -83,6 +87,8 @@ for (const sumFile of sumFiles.sort()) { await run('unzip', ['-o', resolve(inputDir, zipName), '-d', workDir]); const exePath = join(workDir, exeName); const binaryChecksum = await sha256File(exePath); + await copyFile(exePath, resolve(inputDir, binaryName)); + await writeFile(resolve(inputDir, `${binaryName}.sha256`), `${binaryChecksum} ${binaryName}\n`); await run('zstd', ['-T0', '-19', '-q', '-f', '-o', resolve(inputDir, zstName), exePath]); await run('tar', ['-C', workDir, '-czf', resolve(inputDir, tarballName), exeName]); diff --git a/apps/pythinker-code/src/cli/update/native-manifest.ts b/apps/pythinker-code/src/cli/update/native-manifest.ts index b7911b58f..8ff442d0f 100644 --- a/apps/pythinker-code/src/cli/update/native-manifest.ts +++ b/apps/pythinker-code/src/cli/update/native-manifest.ts @@ -1,5 +1,5 @@ /** - * Per-release native artifact manifest (`/binaries//manifest.json`). + * Per-release native artifact manifest (`manifest.json` on the GitHub release). * * Published alongside the release and consumed by the install scripts; the * staged updater reuses the same file so checksums and file names have a @@ -12,7 +12,7 @@ import { valid } from 'semver'; import { z } from 'zod'; -import { pythinkerCodeCdnBinariesBase } from '#/constant/app'; +import { pythinkerCodeReleaseAssetUrl } from '#/constant/app'; const MANIFEST_FETCH_TIMEOUT_MS = 10_000; @@ -47,11 +47,11 @@ export type NativeReleaseManifest = z.infer; export type NativePlatformEntry = z.infer; export function nativeManifestUrl(version: string): string { - return `${pythinkerCodeCdnBinariesBase()}/${version}/manifest.json`; + return pythinkerCodeReleaseAssetUrl(version, 'manifest.json'); } export function nativeBinaryUrl(version: string, filename: string): string { - return `${pythinkerCodeCdnBinariesBase()}/${version}/${filename}`; + return pythinkerCodeReleaseAssetUrl(version, filename); } /** diff --git a/apps/pythinker-code/src/cli/update/native-stage.ts b/apps/pythinker-code/src/cli/update/native-stage.ts index 034a9e259..6af79a6b4 100644 --- a/apps/pythinker-code/src/cli/update/native-stage.ts +++ b/apps/pythinker-code/src/cli/update/native-stage.ts @@ -3,9 +3,10 @@ * without touching the running executable. The actual swap happens on the * next startup (see `native-swap.ts`). * - * The CDN serves the bare platform binary (e.g. `pythinker-code-win32-x64.exe`), - * whose sha256 comes from the per-release manifest over HTTPS — a staged - * binary is byte-exact what the release pipeline produced. + * The GitHub release serves the bare platform binary (e.g. + * `pythinker-code-win32-x64.exe`) and its `.zst` variant, whose sha256 comes + * from the per-release manifest over HTTPS — a staged binary is byte-exact + * what the release pipeline produced. */ import { createHash } from 'node:crypto'; @@ -440,7 +441,7 @@ export async function stageNativeUpdate( // would still be adopted here and reported as success, only for the // startup swap's claim-time re-verify to reject and discard it. Compare // the actual digest before adopting; a mismatch falls through and - // re-stages from the CDN (published under a new generation name — the + // re-stages from the release (published under a new generation name — the // damaged exe is left for the age-gated orphan cleanup). const digest = await hashFileSha256(stagedExePath(options.exePath, existing)); if (digest === existing.sha256) { diff --git a/apps/pythinker-code/src/constant/app.ts b/apps/pythinker-code/src/constant/app.ts index debe14659..500ac38d9 100644 --- a/apps/pythinker-code/src/constant/app.ts +++ b/apps/pythinker-code/src/constant/app.ts @@ -108,11 +108,15 @@ export function pythinkerCodeCdnLatestUrl(): string { export function pythinkerCodeCdnLatestJsonUrl(): string { return `${pythinkerCodeCdnBase()}/latest.json`; } -// Per-release native artifacts: `/binaries//manifest.json` + -// `/binaries//pythinker-code-[.exe]` — the bare platform binary -// (same layout install.ps1 consumes). -export function pythinkerCodeCdnBinariesBase(): string { - return `${pythinkerCodeCdnBase()}/binaries`; +// Per-release native artifacts live on the GitHub release for that version: +// `manifest.json` plus every file it names (bare binary and `.zst`). The +// release pipeline uploads them and `latest.json` points at the same URLs. +// The CDN serves no `/binaries/` route — it answers any unknown path with the +// site's HTML and a 200. +const PYTHINKER_CODE_GITHUB_RELEASES_BASE = 'https://github.com/PyModel/pythinker-code/releases/download'; +export function pythinkerCodeReleaseAssetUrl(version: string, filename: string): string { + const tag = encodeURIComponent(`${NPM_PACKAGE_NAME}@${version}`); + return `${PYTHINKER_CODE_GITHUB_RELEASES_BASE}/${tag}/${filename}`; } // The marketplace env override name lives in the shared agent-core-v2 plugin // domain (agent-gateway consumes it from there). Deep-path import: this module is diff --git a/apps/pythinker-code/test/cli/update/native-manifest.test.ts b/apps/pythinker-code/test/cli/update/native-manifest.test.ts index efb3ecbba..2e9756827 100644 --- a/apps/pythinker-code/test/cli/update/native-manifest.test.ts +++ b/apps/pythinker-code/test/cli/update/native-manifest.test.ts @@ -6,7 +6,6 @@ import { nativeManifestUrl, selectPlatformEntry, } from '#/cli/update/native-manifest'; -import { pythinkerCodeCdnBinariesBase } from '#/constant/app'; const VERSION = '0.7.0'; @@ -195,10 +194,9 @@ describe('selectPlatformEntry', () => { }); describe('url helpers', () => { - it('builds the manifest and binary URLs from the binaries base', () => { - expect(nativeManifestUrl(VERSION)).toBe(`${pythinkerCodeCdnBinariesBase()}/${VERSION}/manifest.json`); - expect(nativeBinaryUrl(VERSION, 'pythinker-code-win32-x64.zip')).toBe( - `${pythinkerCodeCdnBinariesBase()}/${VERSION}/pythinker-code-win32-x64.zip`, - ); + it('points the manifest and binaries at the GitHub release for the version', () => { + const base = `https://github.com/PyModel/pythinker-code/releases/download/%40pymodel%2Fpythinker-code%40${VERSION}`; + expect(nativeManifestUrl(VERSION)).toBe(`${base}/manifest.json`); + expect(nativeBinaryUrl(VERSION, 'pythinker-code-win32-x64.exe')).toBe(`${base}/pythinker-code-win32-x64.exe`); }); }); diff --git a/apps/pythinker-code/test/scripts/native/release-artifacts.test.ts b/apps/pythinker-code/test/scripts/native/release-artifacts.test.ts index 2977c3238..dd5618461 100644 --- a/apps/pythinker-code/test/scripts/native/release-artifacts.test.ts +++ b/apps/pythinker-code/test/scripts/native/release-artifacts.test.ts @@ -97,6 +97,8 @@ describe('native release artifacts', () => { for (const name of [ `pythinker-code-${target}.zip`, `pythinker-code-${target}.zip.sha256`, + `pythinker-code-${target}`, + `pythinker-code-${target}.sha256`, `pythinker-code-${target}.zst`, `pythinker-code-${target}.zst.sha256`, `pythinker-code-${target}.tar.gz`, @@ -168,6 +170,11 @@ describe('native release artifacts', () => { }, }, }); + const bare = resolve(artifactsDir, `pythinker-code-${target}`); + expect(readFileSync(bare, 'utf-8')).toBe(binaryContent); + expect(readFileSync(`${bare}.sha256`, 'utf-8')).toBe( + `${sha256(Buffer.from(binaryContent))} pythinker-code-${target}\n`, + ); }); it('keeps the .exe suffix in Windows manifest filenames', async () => { @@ -199,6 +206,7 @@ describe('native release artifacts', () => { expect(entry.filename).toBe('pythinker-code-win32-x64.exe'); expect(entry.checksum).toBe(sha256(binaryContent)); expect(entry.compressed.filename).toBe('pythinker-code-win32-x64.zst'); + expect(await readFile(join(releaseDir, entry.filename))).toEqual(binaryContent); } finally { rmSync(releaseDir, { recursive: true, force: true }); } diff --git a/scripts/release/cdn-consistency.mjs b/scripts/release/cdn-consistency.mjs index 5b524a664..6adaf8a4d 100644 --- a/scripts/release/cdn-consistency.mjs +++ b/scripts/release/cdn-consistency.mjs @@ -114,3 +114,48 @@ export async function pollCdnUntilCaughtUp(options) { await sleep(intervalMs); } } + +/** + * Every download URL a client can be sent to for `version`: each + * `platforms[*].url` in the CDN `latest.json`, plus every file the release + * `manifest.json` names, resolved against the release asset base. + * + * A matching version string proves nothing about these: 2.4.0 and 2.4.1 + * shipped with the CDN in sync while every bare-binary URL returned 404. + */ +export function collectReleaseDownloadUrls({ latestJson, releaseManifest, releaseAssetUrl }) { + const urls = new Set(); + for (const entry of Object.values(latestJson?.platforms ?? {})) { + if (typeof entry?.url === 'string') urls.add(entry.url); + } + for (const entry of Object.values(releaseManifest?.platforms ?? {})) { + for (const name of [entry?.filename, entry?.compressed?.filename, entry?.zstd?.file]) { + if (typeof name === 'string') urls.add(releaseAssetUrl(name)); + } + } + return [...urls].sort((left, right) => left.localeCompare(right)); +} + +/** + * HEAD each URL and return the ones that do not answer 2xx. A transport error, + * 5xx, 403 or 429 (GitHub rate limiting) is retried `attempts` times in total; + * any other 4xx is final at once. + */ +export async function findUnreachableUrls({ fetchImpl, sleep, urls, attempts = 3, retryDelayMs = 5_000 }) { + const unreachable = []; + for (const url of urls) { + let status = 'unreachable'; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + const response = await fetchImpl(url, { method: 'HEAD' }); + status = response.status; + if (response.ok || (status >= 400 && status < 500 && status !== 403 && status !== 429)) break; + } catch (error) { + status = error instanceof Error ? error.message : 'unreachable'; + } + if (attempt < attempts) await sleep(retryDelayMs); + } + if (typeof status !== 'number' || status < 200 || status >= 300) unreachable.push({ url, status }); + } + return unreachable; +} diff --git a/scripts/release/cdn-consistency.test.mjs b/scripts/release/cdn-consistency.test.mjs new file mode 100644 index 000000000..f5ca9e0ad --- /dev/null +++ b/scripts/release/cdn-consistency.test.mjs @@ -0,0 +1,102 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { collectReleaseDownloadUrls, findUnreachableUrls } from './cdn-consistency.mjs'; + +const BASE = 'https://github.com/PyModel/pythinker-code/releases/download/%40pymodel%2Fpythinker-code%402.4.1'; +const releaseAssetUrl = (name) => `${BASE}/${name}`; + +void test('collects latest.json urls and every file the release manifest names', () => { + const urls = collectReleaseDownloadUrls({ + latestJson: { + version: '2.4.1', + platforms: { 'darwin-arm64': { url: `${BASE}/pythinker-code-darwin-arm64`, sha256: 'a' } }, + }, + releaseManifest: { + platforms: { + 'darwin-arm64': { + filename: 'pythinker-code-darwin-arm64', + compressed: { filename: 'pythinker-code-darwin-arm64.zst' }, + }, + 'win32-x64': { filename: 'pythinker-code-win32-x64.exe', zstd: { file: 'pythinker-code-win32-x64.zst' } }, + }, + }, + releaseAssetUrl, + }); + assert.deepEqual(urls, [ + `${BASE}/pythinker-code-darwin-arm64`, + `${BASE}/pythinker-code-darwin-arm64.zst`, + `${BASE}/pythinker-code-win32-x64.exe`, + `${BASE}/pythinker-code-win32-x64.zst`, + ]); +}); + +void test('collects nothing from manifests without platforms', () => { + assert.deepEqual(collectReleaseDownloadUrls({ latestJson: {}, releaseManifest: {}, releaseAssetUrl }), []); +}); + +void test('reports a 404 at once and passes a 200', async () => { + const calls = []; + const unreachable = await findUnreachableUrls({ + fetchImpl: async (url, init) => { + calls.push([url, init.method]); + return new Response(null, { status: url.endsWith('.zst') ? 200 : 404 }); + }, + sleep: async () => {}, + urls: ['a.zst', 'a'], + }); + assert.deepEqual(unreachable, [{ url: 'a', status: 404 }]); + assert.deepEqual(calls, [ + ['a.zst', 'HEAD'], + ['a', 'HEAD'], + ]); +}); + +void test('retries transport errors and 5xx, then reports the last failure', async () => { + let calls = 0; + const sleeps = []; + const unreachable = await findUnreachableUrls({ + fetchImpl: async () => { + calls += 1; + if (calls === 1) throw new Error('socket hang up'); + return new Response(null, { status: 503 }); + }, + sleep: async (ms) => { + sleeps.push(ms); + }, + urls: ['a'], + attempts: 3, + retryDelayMs: 10, + }); + assert.equal(calls, 3); + assert.deepEqual(sleeps, [10, 10]); + assert.deepEqual(unreachable, [{ url: 'a', status: 503 }]); +}); + +void test('recovers when a retry succeeds', async () => { + let calls = 0; + const unreachable = await findUnreachableUrls({ + fetchImpl: async () => { + calls += 1; + return new Response(null, { status: calls === 1 ? 502 : 200 }); + }, + sleep: async () => {}, + urls: ['a'], + }); + assert.deepEqual(unreachable, []); + assert.equal(calls, 2); +}); + +void test('retries a rate-limited 429 instead of reporting it at once', async () => { + let calls = 0; + const unreachable = await findUnreachableUrls({ + fetchImpl: async () => { + calls += 1; + return new Response(null, { status: calls === 1 ? 429 : 200 }); + }, + sleep: async () => {}, + urls: ['a'], + }); + assert.deepEqual(unreachable, []); + assert.equal(calls, 2); +}); diff --git a/scripts/release/update-brew-formula.mjs b/scripts/release/update-brew-formula.mjs index c725cbbb3..ef1c2af3c 100644 --- a/scripts/release/update-brew-formula.mjs +++ b/scripts/release/update-brew-formula.mjs @@ -3,7 +3,10 @@ * tarball. `changeset publish` can succeed several minutes before the public * GET of `/-/pythinker-code-.tgz` returns 200, so the download polls * until the tarball is fetchable (fetch, sleep, and clock are injected so the - * poll is unit-testable without a network or a real wait). + * poll is unit-testable without a network or a real wait). The poll is also + * the only "is it on npm" gate: `npm view` lags the same way (2.4.0 and 2.4.1 + * both failed a one-shot check that ran seconds after publish), and a version + * that never appears still fails closed once the budget runs out. */ import { createHash } from 'node:crypto'; import { execFileSync, spawnSync } from 'node:child_process'; @@ -55,28 +58,9 @@ export async function downloadNpmTarball(options) { } } -export function assertPublishedNpmVersion({ name, version, execFile = execFileSync }) { - try { - const out = execFile( - 'npm', - ['view', `${name}@${version}`, 'version', '--registry=https://registry.npmjs.org'], - { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }, - ).trim(); - if (out !== version) { - throw new Error(`npm view returned ${out}`); - } - } catch (error) { - throw new Error( - `${name}@${version} is not on npm. Identity freeze: refuse to poll a tarball that will never appear.`, - { cause: error }, - ); - } -} - async function main() { const packageJson = JSON.parse(readFileSync(new URL('../../apps/pythinker-code/package.json', import.meta.url), 'utf8')); const version = packageJson.version; - assertPublishedNpmVersion({ name: '@pymodel/pythinker-code', version }); const tarballUrl = `https://registry.npmjs.org/@pymodel/pythinker-code/-/pythinker-code-${version}.tgz`; const { tarball } = await downloadNpmTarball({ url: tarballUrl, diff --git a/scripts/release/update-brew-formula.test.mjs b/scripts/release/update-brew-formula.test.mjs index a5f66b023..cf8bd7485 100644 --- a/scripts/release/update-brew-formula.test.mjs +++ b/scripts/release/update-brew-formula.test.mjs @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { assertPublishedNpmVersion, downloadNpmTarball } from './update-brew-formula.mjs'; +import { downloadNpmTarball } from './update-brew-formula.mjs'; const TARBALL_URL = 'https://registry.npmjs.org/@pymodel/pythinker-code/-/pythinker-code-2.0.0.tgz'; const BODY = Buffer.from('pythinker-tarball'); @@ -36,31 +36,6 @@ function pollOptions(clock, fetchImpl) { }; } -void test('assertPublishedNpmVersion fails closed when npm does not have the version', () => { - assert.throws( - () => - assertPublishedNpmVersion({ - name: '@pymodel/pythinker-code', - version: '0.43.0', - execFile: () => { - throw new Error('404 Not Found'); - }, - }), - /is not on npm/, - ); -}); - -void test('assertPublishedNpmVersion accepts a matching npm view', () => { - assert.equal( - assertPublishedNpmVersion({ - name: '@pymodel/pythinker-code', - version: '2.1.0', - execFile: () => '2.1.0\n', - }), - undefined, - ); -}); - void test('returns the tarball when the first fetch is HTTP 200', async () => { const clock = pollClock(); let fetches = 0; diff --git a/scripts/release/verify-release-consistency.mjs b/scripts/release/verify-release-consistency.mjs index 1f7368822..0f5c5ddf7 100644 --- a/scripts/release/verify-release-consistency.mjs +++ b/scripts/release/verify-release-consistency.mjs @@ -1,7 +1,7 @@ import { execFileSync } from 'node:child_process'; import { readFileSync } from 'node:fs'; -import { pollCdnUntilCaughtUp } from './cdn-consistency.mjs'; +import { collectReleaseDownloadUrls, findUnreachableUrls, pollCdnUntilCaughtUp } from './cdn-consistency.mjs'; const PACKAGE_NAME = '@pymodel/pythinker-code'; const SEMVER = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:0|[1-9]\d*|\d*[A-Za-z-][0-9A-Za-z-]*))*))?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/; @@ -134,4 +134,41 @@ console.log( `${cdnPoll.retriggers} rebuild request(s)`, ); +// Clients download what latest.json and the release manifest name, so every +// one of those URLs must resolve. A version match alone let releases ship +// whose binaries 404ed for every installed native client. +const releaseAssetUrl = (name) => + `https://github.com/PyModel/pythinker-code/releases/download/${encodeURIComponent(releaseTag)}/${name}`; +async function fetchJson(url) { + const response = await fetch(url, { signal: AbortSignal.timeout(15_000) }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + return JSON.parse(await response.text()); +} +let latestJson; +let releaseManifest; +try { + latestJson = await fetchJson(CDN_MANIFEST_URL); + releaseManifest = await fetchJson(releaseAssetUrl('manifest.json')); +} catch (error) { + fail(`cannot read latest.json or the ${releaseTag} manifest.json: ${error.message}`); +} +const downloadUrls = collectReleaseDownloadUrls({ latestJson, releaseManifest, releaseAssetUrl }); +if (downloadUrls.length === 0) fail(`latest.json and the ${releaseTag} manifest.json name no downloads`); +const unreachable = await findUnreachableUrls({ + fetchImpl: (url, init) => fetch(url, { ...init, signal: AbortSignal.timeout(15_000) }), + sleep: (ms) => + new Promise((resolve) => { + setTimeout(resolve, ms); + }), + urls: downloadUrls, +}); +if (unreachable.length > 0) { + fail( + `${unreachable.length} of ${downloadUrls.length} advertised download(s) do not resolve — ` + + 'native updates for those platforms fail:\n' + + unreachable.map(({ url, status }) => ` ${status} ${url}`).join('\n'), + ); +} +console.log(`All ${downloadUrls.length} advertised downloads resolve`); + console.log(`consistency OK: latest=${distTags.latest} beta=${distTags.beta ?? '-'} dev=${distTags.dev ?? '-'}`);