From 2bdde49703c115ffb6676389c1a65749d19ff7bd Mon Sep 17 00:00:00 2001 From: Ivo Horak Date: Tue, 8 Sep 2026 14:57:12 +0200 Subject: [PATCH 1/3] feat: add bali support --- .github/workflows/check.bali.yml | 173 +++++++ .github/workflows/on.bali-release.yml | 140 ++++++ .github/workflows/on.pr.yml | 28 ++ BALI.md | 290 ++++++++++++ README.md | 12 + bin/bali-docker.test.ts | 101 ++++ bin/bali-docker.ts | 218 +++++++++ bin/bali-release.test.ts | 60 +++ bin/bali-release.ts | 69 +++ bin/docker.ts | 190 ++++++++ bin/run.ts | 241 +++------- docker/bali.Dockerfile | 22 + expectations/jdk-jtreg.toml | 12 + harness/src/adapters/index.ts | 2 + harness/src/adapters/javac-jtreg.ts | 36 +- harness/src/adapters/jdk-jtreg.test.ts | 320 +++++++++++++ harness/src/adapters/jdk-jtreg.ts | 624 +++++++++++++++++++++++++ harness/src/adapters/jtreg.ts | 53 +++ harness/src/adapters/types.ts | 5 +- harness/src/cli.ts | 31 +- harness/src/registry.test.ts | 10 + harness/src/registry.ts | 3 + harness/src/results/schema.ts | 12 +- harness/src/target.ts | 24 + harness/src/targets.test.ts | 17 + manifests/bali-jdk.json | 20 + registry.toml | 13 + reports/bali/index.json | 3 + reports/bali/index.md | 3 + 29 files changed, 2518 insertions(+), 214 deletions(-) create mode 100644 .github/workflows/check.bali.yml create mode 100644 .github/workflows/on.bali-release.yml create mode 100644 BALI.md create mode 100644 bin/bali-docker.test.ts create mode 100644 bin/bali-docker.ts create mode 100644 bin/bali-release.test.ts create mode 100644 bin/bali-release.ts create mode 100644 bin/docker.ts create mode 100644 docker/bali.Dockerfile create mode 100644 expectations/jdk-jtreg.toml create mode 100644 harness/src/adapters/jdk-jtreg.test.ts create mode 100644 harness/src/adapters/jdk-jtreg.ts create mode 100644 harness/src/adapters/jtreg.ts create mode 100644 harness/src/target.ts create mode 100644 harness/src/targets.test.ts create mode 100644 manifests/bali-jdk.json create mode 100644 reports/bali/index.json create mode 100644 reports/bali/index.md diff --git a/.github/workflows/check.bali.yml b/.github/workflows/check.bali.yml new file mode 100644 index 0000000..a64c166 --- /dev/null +++ b/.github/workflows/check.bali.yml @@ -0,0 +1,173 @@ +name: "Check - Bali Compatibility" + +on: + workflow_call: + secrets: + testsuite_token: + description: "Token for a private testsuite checkout from another repository; needs contents and pull-request write access when apply_updates is set" + required: false + inputs: + testsuite_ref: + description: "Reviewed testsuite commit containing the Bali integration" + required: true + type: string + artifact: + description: "Bali distribution artifact from the calling build" + required: true + type: string + runner: + required: false + default: ubuntu-24.04 + type: string + ratchet: + description: "Regenerate expectations/jdk-jtreg.ratchet.toml from this run's failures, like Elide's ratchet" + required: false + default: false + type: boolean + apply_updates: + description: "Commit reports/bali to update_branch in the testsuite repository and open a pull request" + required: false + default: false + type: boolean + create_pr: + description: "Open or refresh a pull request for update_branch when apply_updates commits changes" + required: false + default: true + type: boolean + update_branch: + description: "Testsuite branch receiving committed reports" + required: false + default: "sync/bali-compatibility" + type: string + update_base: + description: "Testsuite branch the report pull request targets" + required: false + default: "main" + type: string + +permissions: + contents: read + actions: read + +jobs: + compatibility: + runs-on: ${{ inputs.runner }} + timeout-minutes: 60 + steps: + - uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 + with: + disable-sudo: true + egress-policy: audit + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + repository: elide-dev/testsuite + token: ${{ secrets.testsuite_token || github.token }} + ref: ${{ inputs.testsuite_ref }} + persist-credentials: false + - uses: step-security/setup-bun@f6f5dadeac34f70c7828f731569e8d6e8330b8fb + with: + bun-version: "1.4.0" + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: ${{ inputs.artifact }} + path: .harness/distribution-artifact + - name: Extract distribution + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + archives=(.harness/distribution-artifact/bali-*.tgz) + [[ ${#archives[@]} -eq 1 ]] || { echo 'Expected exactly one Bali distribution tarball'; exit 2; } + mkdir -p .harness/distribution + tar -xzf "${archives[0]}" -C .harness/distribution + - name: Run OpenJDK runtime tests + env: + RATCHET: ${{ inputs.ratchet }} + shell: bash + run: | + set -euo pipefail + args=(--bali-home "$PWD/.harness/distribution") + if [[ "$RATCHET" == "true" ]]; then args+=(--ratchet); fi + bun run testsuite --target bali "${args[@]}" + - name: Job summary + if: always() + shell: bash + run: | + if [[ -f .harness/work/jdk-jtreg/report.md ]]; then + cat .harness/work/jdk-jtreg/report.md >> "$GITHUB_STEP_SUMMARY" + else + echo 'Bali compatibility did not produce a report. See setup/harness logs.' >> "$GITHUB_STEP_SUMMARY" + fi + if [[ -f reports/bali/index.md ]]; then + printf '\n' >> "$GITHUB_STEP_SUMMARY" + cat reports/bali/index.md >> "$GITHUB_STEP_SUMMARY" + fi + - name: Upload results and diagnostics + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: bali-compatibility-${{ runner.os }}-${{ runner.arch }} + include-hidden-files: true + retention-days: 30 + path: | + reports/bali/** + expectations/jdk-jtreg.ratchet.toml + .harness/work/jdk-jtreg/report.* + .harness/work/jdk-jtreg/inventory.json + .harness/work/jdk-jtreg/jtreg-run-*/reference/** + .harness/work/jdk-jtreg/jtreg-run-*/bali/** + if-no-files-found: warn + - name: "Update: Commit Generated Reports" + if: ${{ always() && inputs.apply_updates }} + id: update + shell: bash + env: + TESTSUITE_TOKEN: ${{ secrets.testsuite_token }} + UPDATE_BRANCH: ${{ inputs.update_branch || 'sync/bali-compatibility' }} + run: | + set -euo pipefail + if [[ -z "$TESTSUITE_TOKEN" ]]; then + echo '::error::apply_updates needs the testsuite_token secret with contents and pull-request write access to elide-dev/testsuite; the calling job token cannot push to another repository.' + exit 2 + fi + git config user.name "elide-ci" + git config user.email "ci@elide.dev" + git add -- reports/bali expectations/jdk-jtreg.ratchet.toml + if git diff --cached --quiet; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "No generated changes to commit." + exit 0 + fi + + git checkout -B "$UPDATE_BRANCH" + git commit -m "ci: update Bali compatibility reports" + # Checkout ran without persisted credentials; authenticate git only here. + auth="$(printf 'x-access-token:%s' "$TESTSUITE_TOKEN" | base64 | tr -d '\n')" + header="http.https://github.com/.extraheader=AUTHORIZATION: basic $auth" + # The shallow checkout has no remote-tracking ref for the update branch, so + # fetch it first; otherwise --force-with-lease rejects any existing branch. + git -c "$header" fetch --no-tags origin \ + "+refs/heads/$UPDATE_BRANCH:refs/remotes/origin/$UPDATE_BRANCH" || true + git -c "$header" push --force-with-lease origin "$UPDATE_BRANCH" + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "branch=$UPDATE_BRANCH" >> "$GITHUB_OUTPUT" + + - name: "Update: Open Pull Request" + if: ${{ always() && inputs.apply_updates && inputs.create_pr && steps.update.outputs.changed == 'true' }} + shell: bash + env: + GH_TOKEN: ${{ secrets.testsuite_token }} + BRANCH: ${{ steps.update.outputs.branch }} + BASE_BRANCH: ${{ inputs.update_base || 'main' }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + title="ci: update Bali compatibility reports" + body="Automated Bali compatibility report update from ${GITHUB_WORKFLOW} run ${GITHUB_RUN_ID} ($RUN_URL). This records the measurement under reports/bali/ and includes the ratchet file when the run regenerated it." + existing="$(gh pr list --repo elide-dev/testsuite --head "$BRANCH" --state open --json url --jq '.[0].url // ""')" + if [[ -n "$existing" ]]; then + gh pr edit "$existing" --repo elide-dev/testsuite --title "$title" --body "$body" + echo "Updated PR: $existing" + else + gh pr create --repo elide-dev/testsuite --base "$BASE_BRANCH" --head "$BRANCH" --title "$title" --body "$body" + fi diff --git a/.github/workflows/on.bali-release.yml b/.github/workflows/on.bali-release.yml new file mode 100644 index 0000000..8464942 --- /dev/null +++ b/.github/workflows/on.bali-release.yml @@ -0,0 +1,140 @@ +name: "Manual - Bali Release Compatibility" + +"on": + workflow_dispatch: + inputs: + ratchet: + description: "Regenerate expectations/jdk-jtreg.ratchet.toml from this run's failures, like Elide's ratchet" + required: false + default: false + type: boolean + # Add a schedule here when recurring measurements are wanted. The job's + # defaults also work without workflow_dispatch inputs. + +permissions: + contents: read + +concurrency: + group: bali-release-compatibility + cancel-in-progress: false + +jobs: + compatibility: + name: "Latest Bali release / Linux AMD64" + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + contents: write + pull-requests: write + steps: + - name: Harden runner + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 + with: + disable-sudo: true + egress-policy: audit + - name: Checkout testsuite + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: true + - name: Setup Bun + uses: step-security/setup-bun@f6f5dadeac34f70c7828f731569e8d6e8330b8fb + with: + bun-version: "1.4.0" + - name: Resolve and download latest stable release + env: + # Bali is internal: the testsuite job token cannot read another repository. + GH_TOKEN: ${{ secrets.BALI_RELEASE_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "$GH_TOKEN" ]]; then + echo '::error::Configure BALI_RELEASE_TOKEN with contents:read access to elide-dev/bali in the testsuite repository or organization.' + exit 2 + fi + mkdir -p .harness/release .harness/distribution + gh api repos/elide-dev/bali/releases/latest > .harness/release/release.json + bun bin/bali-release.ts select .harness/release/release.json .harness/release/selection.json + asset_id=$(bun -e 'console.log((await Bun.file(".harness/release/selection.json").json()).assetId)') + # Download the resolved asset ID, never resolve "latest" a second time. + gh api "repos/elide-dev/bali/releases/assets/$asset_id" \ + -H 'Accept: application/octet-stream' > .harness/release/distribution.tgz + bun bin/bali-release.ts verify .harness/release/selection.json .harness/release/distribution.tgz + tar -xzf .harness/release/distribution.tgz -C .harness/distribution + - name: Run OpenJDK compatibility tests + env: + RATCHET: ${{ inputs.ratchet || false }} + shell: bash + run: | + set -euo pipefail + args=(--bali-home "$PWD/.harness/distribution") + if [[ "$RATCHET" == "true" ]]; then args+=(--ratchet); fi + bun run testsuite --target bali "${args[@]}" + - name: Write job summary + if: always() + shell: bash + run: | + { + echo '## Bali release compatibility' + if [[ -f .harness/release/selection.json ]]; then + echo 'Resolved release and expected archive checksum:' + echo '```json' + cat .harness/release/selection.json + echo '```' + fi + if [[ -f .harness/work/jdk-jtreg/report.md ]]; then + cat .harness/work/jdk-jtreg/report.md + else + echo 'No test report was produced. Inspect the setup and harness logs.' + fi + if [[ -f reports/bali/index.md ]]; then + cat reports/bali/index.md + fi + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload reports and diagnostics + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: bali-release-compatibility-linux-amd64 + include-hidden-files: true + retention-days: 30 + if-no-files-found: warn + path: | + .harness/release/release.json + .harness/release/selection.json + .harness/work/jdk-jtreg/report.* + .harness/work/jdk-jtreg/inventory.json + .harness/work/jdk-jtreg/jtreg-run-*/reference/** + .harness/work/jdk-jtreg/jtreg-run-*/bali/** + expectations/jdk-jtreg.ratchet.toml + reports/bali/** + - name: Save report history in a pull request + if: ${{ always() && !cancelled() && github.ref_type == 'branch' && hashFiles('.harness/work/jdk-jtreg/report.json') != '' }} + env: + GH_TOKEN: ${{ github.token }} + BASE_BRANCH: ${{ github.ref_name }} + REPORT_BRANCH: sync/bali-reports-${{ github.run_id }}-${{ github.run_attempt }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + shell: bash + run: | + set -euo pipefail + git config user.name "elide-ci" + git config user.email "ci@elide.dev" + git add -- reports/bali expectations/jdk-jtreg.ratchet.toml + if git diff --cached --quiet; then + echo 'No new Bali reports to commit.' + exit 0 + fi + # Each run keeps its own branch so unmerged measurements are retained. + git checkout -b "$REPORT_BRANCH" + git commit -m "ci: record Bali compatibility measurement" + git push origin "$REPORT_BRANCH" + body_file="$RUNNER_TEMP/bali-report-pr.md" + cat > "$body_file" <//jdk-jtreg/` +has exactly the Elide file set (`results.json.gz`, `summary.json`, `jdk-jtreg.md`, +`index.md`, `pass-rate.svg`, `impact.*`, `changes.*`) plus two adapter extras: +`differential.md`, the paired view rebuilt from each result's metadata, and +`coverage.svg`. `reports/bali/index.json`, `index.md`, and `pass-rate.svg` are the +same generated indexes Elide keeps under `reports/`. + +Raw logs and local paths stay in ignored run directories. The reusable +`check.bali.yml` workflow uploads raw diagnostics and generated reports even after +test failures, and can commit `reports/bali/` and the ratchet file when the caller +opts in with `apply_updates`. + +The source is Java 25 GA at commit +`6c48f4ed707bf0b15f9b6098de30db8aae6fa40f`. The jtreg archive is Shipilev's +checksum-pinned `7.5.1+1` rebuild, reporting `7.5.1-dev+0`; it is not an Oracle +binary. Downloads are verified on every use and cached under +`.harness/bali/archives`. Source or archive updates require updating the pins +together and re-ratcheting. + +## Shared repository integration + +`jdk-jtreg` is a regular `[[workload]]` in `registry.toml` with `target = "bali"` and +`adapter = "jdk-jtreg"`; the adapter is registered beside the Elide adapters and the +manifest path comes from the workload settings. It shares jtreg mechanics (run-root +creation, common flags, outcome names, runner exit-code rules) with the javac adapter +through `harness/src/adapters/jtreg.ts`; what remains Bali-specific is running the +corpus on two JDKs and flattening the pair. `bun run testsuite --target bali` runs +it; the Elide launcher skips workloads with another target, and `--all-suites` never +includes it. Elide remains the default target; its commands and history are unchanged. +The shared launcher runs Bali in Linux AMD64 Docker, without an Elide installation. +Use `--execution native` explicitly for native platform testing. + +The harness identifies a run by the Bali version (from `bin/java -version`) and the +digest of the distribution files, recorded as the run's target in `summary.json`. +Re-measuring the same binary replaces its report, and `changes.*` diffs against the +previous run of the workload, exactly as for Elide builds. Because Bali reports live +under `reports/bali/`, they have their own indexes and are not part of the Elide README +table or SQL commands. + +## Calling from Bali CI + +After publishing this integration, call +`elide-dev/testsuite/.github/workflows/check.bali.yml@` from the +Bali build workflow, depending on its existing distribution build. Supply the +same reviewed commit as `testsuite_ref`, the uploaded distribution artifact name +as `artifact`, and a compatible runner. Artifacts are downloaded from the calling +workflow run; the native image is never rebuilt here. For a private testsuite +checkout, pass the optional `testsuite_token` secret with read access to this +repository. + +The expectations file starts empty, so the first runs return exit 1 for every +Bali-only failure while still uploading results and writing `reports/bali/`. Run once with `ratchet: true`, +review the generated `expectations/jdk-jtreg.ratchet.toml`, and commit it; after +that the job is green unless a previously passing file regresses. Establish the +ratchet on the actual CI runner before making the job required. The workflow is +reusable, not scheduled; no remote workflow or publication was enabled by adding it. + +By default the reusable workflow stores results only in the workflow artifact. +Pass `apply_updates: true` to also commit `reports/bali/` and the ratchet file to +the testsuite repository, following Elide's compliance workflow: the job checks +out `testsuite_ref`, publishes the measurement, commits to `update_branch` +(default `sync/bali-compatibility`) with a force push, and opens or refreshes a +pull request against `update_base` (default `main`) unless `create_pr` is false. +The commit runs even when tests fail, so red measurements are retained. A +later run replaces an unmerged update branch, the same as Elide's +`sync/compliance`; merge report PRs promptly if every measurement matters, or +use the manual release workflow, which keeps one branch per run. Because the +calling repository's job token cannot push to testsuite, `apply_updates` +requires the `testsuite_token` secret with contents and pull-request write +access to this repository; the job fails with an explicit error otherwise. +Hand-curated expectations are never modified by CI; only the ratchet file is. + +## Manual latest-release measurements + +In this repository's Actions tab, choose **Manual - Bali Release Compatibility** +and **Run workflow**. It resolves GitHub's latest published, non-prerelease Bali +release once, downloads its Linux AMD64 tarball by asset ID, verifies the API's +SHA-256 digest, and tests it in a Linux AMD64 Docker container on Ubuntu 24.04. +No Bali checkout, native-image build, or Elide installation is needed. + +Leave `ratchet` unchecked for an ordinary measurement; existing regressions +return exit 1 while still producing reports. Check it to regenerate the ratchet +file from the run, which the report PR then includes for review. Incomplete runs +fail rather than becoming green. + +The job summary shows the selected release and test results. The +`bali-release-compatibility-linux-amd64` artifact retains release provenance, +per-test results, generated history, and raw jtreg diagnostics for 30 days, +including when testing fails. + +For runs dispatched from a branch, the workflow also commits `reports/bali/` and +the ratchet file to `sync/bali-reports--` and opens a PR against +the dispatch branch. Each run has its own branch so later runs cannot overwrite +unmerged measurements. Merge the PR to retain the results in the repository and +expose them through `reports/bali/index.md`. Concurrent report PRs can conflict in +the generated indexes; resolve that as for Elide's `reports/index.*`, by keeping +both measurement directories and letting the next run regenerate the indexes. +Publishing does not turn a failed check green. Runs without a +measurement cannot publish; tag dispatches retain artifacts only. + +Report publication uses the ordinary `GITHUB_TOKEN` with contents and pull-request +write permissions. The repository must allow GitHub Actions to create pull requests. +Bali is currently an internal repository: configure `BALI_RELEASE_TOKEN` in +testsuite (or as an organization secret accessible to testsuite) with read access +to Bali's contents. The workflow reports an actionable error if it is missing. + +Only manual dispatch is enabled. A future `schedule` trigger can use the same +job unchanged; no required dispatch inputs or interactive prompts are involved. + +## Docker execution in CI + +Bali uses the same execution model as Elide: the shared launcher builds a +harness image and runs `harness/src/cli.ts` in Docker. `--target bali` defaults to +**linux/amd64**. Both Bali CI workflows use this path. The image uses Debian Trixie +like Elide's harness, pinned Bun 1.4.0, stock Temurin JDK 25.0.2 as the reference, +and the harness dependencies. Both base images are pinned by digest. + +```sh +bun run testsuite --target bali --bali-home /path/to/linux-amd64/bali +``` + +The distribution is mounted read-only at `/opt/bali`; `registry.toml` and +`manifests/` are mounted read-only; `expectations/` is mounted read-only unless +`--ratchet` is given; `reports/` and `.harness/` are writable, exactly as for Elide +runs. Docker runs as the invoking user, with the same user mapping, writable-mount +checks, and interrupt cleanup as the Elide launcher, shared through `bin/docker.ts`. +Release access credentials stay on the host and are not passed into the test +container. Like Elide, no image or checkout provenance is recorded beyond the run +identity. No macOS measurement is published. + +`--execution native` runs the same harness entry point on the host. CI uses the +Docker path. diff --git a/README.md b/README.md index 21f3579..80d87ad 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,18 @@ Tracking: [Compliance Testing meta (WHIPLASH#1172)](https://github.com/elide-dev > Full enabled slices may be RED until expectations are ratcheted and failures > are classified. +## Bali runtime compatibility + +This repository also supports Bali as a Docker-based Linux AMD64 target. Elide remains the default. + +```sh +bun run testsuite --target bali --suite jdk-jtreg --bali-home /path/to/linux-amd64/bali +``` + +See [Bali setup and expectations](BALI.md) and [Bali compatibility history](reports/bali/index.md). +The runtime suite (`test/jdk`) is separate from Elide's compiler suite (`javac-jtreg`). +Runner coverage, verified passes, reference issues, and unsupported files remain separate. + ## Compatibility diff --git a/bin/bali-docker.test.ts b/bin/bali-docker.test.ts new file mode 100644 index 0000000..1314155 --- /dev/null +++ b/bin/bali-docker.test.ts @@ -0,0 +1,101 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, mkdir, rm } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { baliPlan, containerArgs, harnessArgs, runBaliDocker } from "./bali-docker"; + +const user = ["--user", "1000:1000"]; + +test("Bali plan drives the shared harness with --target bali and never forwards release credentials", () => { + const plan = baliPlan(["--ratchet", "--bali-home", "dist"], "/repo"); + const args = containerArgs(plan, "/repo", "sha256:" + "a".repeat(64), "run=fixture", "digest1", user); + expect(args).toContain("1000:1000"); + expect(args).not.toContain("TESTSUITE_REVISION=abc"); + expect(args).toContain("linux/amd64"); + expect(args).toContain("type=bind,src=/repo/dist,dst=/opt/bali,readonly"); + expect(args).toContain("type=bind,src=/repo/expectations,dst=/work/expectations"); + expect(args).toContain("type=bind,src=/repo/registry.toml,dst=/work/registry.toml,readonly"); + expect(args.slice(args.indexOf("sha256:" + "a".repeat(64)) + 1)).toEqual( + harnessArgs(plan, "digest1", { + registry: "/work/registry.toml", + repoRoot: "/work", + baliHome: "/opt/bali", + suiteRoot: "/work/suites", + reports: "/work/reports/bali", + expectations: "/work/expectations", + }), + ); + expect(args).toContain("--target"); + expect(args).toContain("/work/reports/bali"); + expect(args).toContain("--ratchet"); + expect(args).not.toContain("GH_TOKEN"); + expect(args).not.toContain("BALI_RELEASE_TOKEN"); + const readonly = containerArgs( + baliPlan(["--bali-home", "dist"], "/repo"), + "/repo", + "sha256:" + "a".repeat(64), + "run=fixture", + "digest1", + user, + ); + expect(readonly).toContain("type=bind,src=/repo/expectations,dst=/work/expectations,readonly"); + expect(readonly).not.toContain("--ratchet"); +}); + +test("Bali plan validates suites from the registry and accepts --ratchet anywhere", () => { + expect(baliPlan(["--suite", "jdk-jtreg", "--ratchet", "--bali-home", "dist"], "/repo").ratchet).toBe(true); + expect(baliPlan(["--bali-home", "--ratchet"], "/repo").ratchet).toBe(false); + expect(() => baliPlan(["--ratchet", "--ratchet", "--bali-home", "dist"], "/repo")).toThrow(); + expect(() => baliPlan(["--suite", "test262", "--bali-home", "dist"], "/repo")).toThrow( + "Bali supports --suite jdk-jtreg", + ); + expect(baliPlan(["--bali-home", "dist"], "/repo", ["jdk-jtreg", "other"]).suite).toBe("jdk-jtreg"); + expect(baliPlan(["--reference-home", "/jdk", "--bali-home", "dist"], "/repo").referenceHome).toBe("/jdk"); +}); + +test("Docker launcher rejects macOS artifacts and a host reference JDK, and propagates build errors", async () => { + const temp = await mkdtemp(join(tmpdir(), "bali-container-")); + try { + const home = join(temp, "distribution"); + await mkdir(join(home, "bin"), { recursive: true }); + await mkdir(join(home, "lib"), { recursive: true }); + await Bun.write(join(home, "bin/bali"), new Uint8Array(20)); + for (const file of ["bin/java", "lib/modules", "release"]) await Bun.write(join(home, file), file); + const commands: string[][] = []; + const run = async (args: string[]) => { + commands.push(args); + return 1; + }; + const root = resolve(import.meta.dir, ".."); + const argv = ["--bali-home", home]; + await expect( + runBaliDocker([...argv, "--reference-home", "/jdk"], root, "test=fixture", run), + ).rejects.toThrow("fixed in the image"); + await expect(runBaliDocker(argv, root, "test=fixture", run)).rejects.toThrow("Linux AMD64"); + expect(commands).toHaveLength(0); + const elf = new Uint8Array(20); + elf.set([127, 69, 76, 70, 2, 1]); + elf[18] = 62; + await Bun.write(join(home, "bin/bali"), elf); + expect(await runBaliDocker(argv, root, "test=fixture", run)).toBe(2); + expect(commands).toHaveLength(1); + expect(commands[0]).toContain("linux/amd64"); + const successCommands: string[][] = []; + const regression = await runBaliDocker(argv, root, "test=fixture", async (args) => { + successCommands.push(args); + if (args[1] === "build") { + await Bun.write(args[args.indexOf("--iidfile") + 1]!, "sha256:" + "a".repeat(64)); + return 0; + } + return 1; + }); + expect(regression).toBe(1); + expect(successCommands).toHaveLength(2); + expect(successCommands[1]).toContain("--rm"); + expect(successCommands[1]).toContain("--init"); + expect(successCommands[1]).toContain("jdk-jtreg"); + expect(successCommands[1]).toContain("--digest"); + } finally { + await rm(temp, { recursive: true, force: true }); + } +}); diff --git a/bin/bali-docker.ts b/bin/bali-docker.ts new file mode 100644 index 0000000..59d74bf --- /dev/null +++ b/bin/bali-docker.ts @@ -0,0 +1,218 @@ +// Bali target launcher: runs the shared harness (harness/src/cli.ts) for workloads whose +// registry `target` is "bali", in Linux AMD64 Docker by default or natively on request. +// Docker primitives are shared with the Elide launcher through ./docker. +import { randomUUID } from "node:crypto"; +import { mkdir, realpath, rm } from "node:fs/promises"; +import { resolve, join } from "node:path"; +import { artifactDigest } from "../harness/src/adapters/jdk-jtreg"; +import { + assertWritableMountSources, + platformArgs, + requireDocker, + run as runCommand, + userArgs, +} from "./docker"; + +const PLATFORM = "linux/amd64"; + +export interface BaliPlan { + baliHome: string; + referenceHome?: string; + suite: string; + ratchet: boolean; +} +export function baliPlan(argv: string[], cwd: string, suites: string[] = ["jdk-jtreg"]): BaliPlan { + const args = [...argv]; + let ratchet = false; + const options = new Map(); + for (let i = 0; i < args.length; i += 2) { + if (args[i] === "--ratchet" && !ratchet) { + ratchet = true; + i--; + continue; + } + if ( + !["--bali-home", "--reference-home", "--suite"].includes(args[i]!) || + !args[i + 1] || + options.has(args[i]!) + ) + throw new Error( + `Bali runs accept --bali-home [--reference-home ] [--ratchet] [--suite ${suites.join("|")}].`, + ); + options.set(args[i]!, args[i + 1]!); + } + const suite = options.get("--suite") ?? suites[0]!; + if (!suites.includes(suite)) throw new Error(`Bali supports --suite ${suites.join(", ")}`); + if (!options.has("--bali-home")) + throw new Error("Bali runs require --bali-home pointing to a packaged distribution"); + return { + baliHome: resolve(cwd, options.get("--bali-home")!), + referenceHome: options.has("--reference-home") + ? resolve(cwd, options.get("--reference-home")!) + : undefined, + suite, + ratchet, + }; +} +export interface HarnessPaths { + registry: string; + repoRoot: string; + baliHome: string; + suiteRoot: string; + reports: string; + expectations: string; +} +/** The same cli.ts invocation the Elide launcher builds, with --target bali. */ +export function harnessArgs(plan: BaliPlan, digest: string, paths: HarnessPaths): string[] { + return [ + "run", + plan.suite, + "--target", + "bali", + "--registry", + paths.registry, + "--repo-root", + paths.repoRoot, + "--elide-path", + paths.baliHome, + "--digest", + digest, + "--suite-root", + paths.suiteRoot, + "--reports", + paths.reports, + "--expectations", + paths.expectations, + "--log-prefix", + `[${plan.suite}] `, + "--log", + "--failure-output", + "hide", + ...(plan.ratchet ? ["--ratchet"] : []), + ]; +} +export function containerArgs( + plan: BaliPlan, + root: string, + image: string, + label: string, + digest: string, + user: string[], +): string[] { + return [ + "docker", + "run", + "--rm", + "--init", + ...platformArgs(PLATFORM), + "--label", + label, + ...user, + "--mount", + `type=bind,src=${plan.baliHome},dst=/opt/bali,readonly`, + "--mount", + `type=bind,src=${join(root, ".harness")},dst=/work/.harness`, + "--mount", + `type=bind,src=${join(root, "reports")},dst=/work/reports`, + // Expectations are mounted like Elide's: read-only unless this run ratchets them. + "--mount", + `type=bind,src=${join(root, "expectations")},dst=/work/expectations${plan.ratchet ? "" : ",readonly"}`, + "--mount", + `type=bind,src=${join(root, "registry.toml")},dst=/work/registry.toml,readonly`, + "--mount", + `type=bind,src=${join(root, "manifests")},dst=/work/manifests,readonly`, + image, + ...harnessArgs(plan, digest, { + registry: "/work/registry.toml", + repoRoot: "/work", + baliHome: "/opt/bali", + suiteRoot: "/work/suites", + reports: "/work/reports/bali", + expectations: "/work/expectations", + }), + ]; +} +async function prepareRoot(root: string): Promise { + await mkdir(join(root, ".harness"), { recursive: true }); + await mkdir(join(root, "reports/bali"), { recursive: true }); + await mkdir(join(root, "expectations"), { recursive: true }); +} +export async function runBaliDocker( + argv: string[], + root: string, + label: string, + run: (args: string[]) => Promise = runCommand, + suites?: string[], +): Promise { + const plan = baliPlan(argv, process.cwd(), suites); + if (plan.referenceHome) + throw new Error( + "--reference-home is not accepted for Docker runs: the reference JDK is fixed in the image. Use --execution native.", + ); + plan.baliHome = await realpath(plan.baliHome); + // Reject macOS binaries before spending time building a Linux image. + const magic = new Uint8Array( + await Bun.file(join(plan.baliHome, "bin/bali")).slice(0, 20).arrayBuffer(), + ); + if ( + magic[0] !== 0x7f || + magic[1] !== 0x45 || + magic[2] !== 0x4c || + magic[3] !== 0x46 || + magic[4] !== 2 || + magic[5] !== 1 || + magic[18] !== 62 || + magic[19] !== 0 + ) + throw new Error( + "Docker tests need the Linux AMD64 Bali release, not a macOS/ARM64 distribution. Use --execution native for native platform testing.", + ); + const digest = await artifactDigest(plan.baliHome); + await prepareRoot(root); + assertWritableMountSources(["reports", "expectations", ".harness"]); + if (run === runCommand) await requireDocker(); + const user = userArgs(String(process.getuid?.() ?? 1000), String(process.getgid?.() ?? 1000)); + const dockerfile = join(root, "docker/bali.Dockerfile"); + const iid = join(root, ".harness", `bali-image-${randomUUID()}.txt`); + console.log(`Building Bali test environment (${PLATFORM})...`); + try { + const built = await run([ + "docker", + "build", + ...platformArgs(PLATFORM), + "--iidfile", + iid, + "-f", + dockerfile, + root, + ]); + if (built !== 0) return 2; + const image = (await Bun.file(iid).text()).trim(); + if (!/^sha256:[a-f0-9]{64}$/.test(image)) + throw new Error("Docker did not return a valid image identity"); + console.log(`Running ${plan.suite} in ${image}; workspace: ${join(root, ".harness/work", plan.suite)}`); + return await run(containerArgs(plan, root, image, label, digest, user)); + } finally { + await rm(iid, { force: true }); + } +} +/** Native host run through the same harness entry point, with JAVA_HOME as the reference. */ +export async function runBaliNative(argv: string[], root: string, suites?: string[]): Promise { + const plan = baliPlan(argv, process.cwd(), suites); + const baliHome = await realpath(plan.baliHome); + if (plan.referenceHome) process.env.JAVA_HOME = plan.referenceHome; + await prepareRoot(root); + const { main, parseArgs } = await import("../harness/src/cli"); + return main( + parseArgs( + harnessArgs(plan, await artifactDigest(baliHome), { + registry: join(root, "registry.toml"), + repoRoot: root, + baliHome, + suiteRoot: join(root, "suites"), + reports: join(root, "reports/bali"), + expectations: join(root, "expectations"), + }), + ), + ); +} diff --git a/bin/bali-release.test.ts b/bin/bali-release.test.ts new file mode 100644 index 0000000..74297b0 --- /dev/null +++ b/bin/bali-release.test.ts @@ -0,0 +1,60 @@ +import { expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { selectBaliRelease, verifyReleaseArchive, type Release } from "./bali-release"; +const release = (): Release => ({ + id: 1, + tag_name: "v0.4.0", + html_url: "https://github.com/elide-dev/bali/releases/tag/v0.4.0", + published_at: "2026-08-30T03:44:48Z", + draft: false, + prerelease: false, + assets: [ + { + id: 42, + name: "bali-0.4.0-linux-amd64.tgz", + digest: "sha256:" + "a".repeat(64), + state: "uploaded", + }, + { + id: 43, + name: "bali-0.4.0-linux-arm64.tgz", + digest: "sha256:" + "b".repeat(64), + state: "uploaded", + }, + ], +}); +test("latest release selection binds the exact Linux AMD64 asset and checksum", () => { + expect(selectBaliRelease(release())).toMatchObject({ + tag: "v0.4.0", + assetId: 42, + sha256: "a".repeat(64), + platform: "linux-amd64", + }); + for (const bad of [ + { ...release(), draft: true }, + { ...release(), prerelease: true }, + { ...release(), assets: [] }, + { ...release(), tag_name: "v0.4.0\ninjected" }, + ]) + expect(() => selectBaliRelease(bad)).toThrow(); + const duplicate = release(); + duplicate.assets.push(duplicate.assets[0]!); + expect(() => selectBaliRelease(duplicate)).toThrow("exactly one"); + const missing = release(); + missing.assets[0]!.digest = null; + expect(() => selectBaliRelease(missing)).toThrow("SHA-256"); +}); +test("archive verification rejects corrupted downloads", async () => { + const root = await mkdtemp(join(tmpdir(), "bali-release-")); + try { + const path = join(root, "distribution.tgz"); + await Bun.write(path, "downloaded bytes"); + await verifyReleaseArchive(path, createHash("sha256").update("downloaded bytes").digest("hex")); + await expect(verifyReleaseArchive(path, "a".repeat(64))).rejects.toThrow("mismatch"); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/bin/bali-release.ts b/bin/bali-release.ts new file mode 100644 index 0000000..265276d --- /dev/null +++ b/bin/bali-release.ts @@ -0,0 +1,69 @@ +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; + +interface ReleaseAsset { + id: number; + name: string; + digest: string | null; + state: string; +} +export interface Release { + id: number; + tag_name: string; + html_url: string; + published_at: string; + draft: boolean; + prerelease: boolean; + assets: ReleaseAsset[]; +} +export function selectBaliRelease(release: Release) { + if (release.draft || release.prerelease || !release.published_at) + throw new Error("Expected a published stable Bali release"); + if (!/^v?\d+\.\d+\.\d+(?:[.+-][0-9A-Za-z.-]+)?$/.test(release.tag_name)) + throw new Error("Unexpected Bali release tag"); + const name = `bali-${release.tag_name.replace(/^v/, "")}-linux-amd64.tgz`; + const assets = release.assets.filter((asset) => asset.name === name); + if (assets.length !== 1) throw new Error(`Expected exactly one release asset: ${name}`); + const asset = assets[0]!; + if (!Number.isSafeInteger(asset.id) || asset.id <= 0 || asset.state !== "uploaded") + throw new Error("Release asset is not ready for download"); + if (!asset.digest || !/^sha256:[a-f0-9]{64}$/.test(asset.digest)) + throw new Error("Release asset has no usable SHA-256 digest"); + return { + repository: "elide-dev/bali", + releaseId: release.id, + tag: release.tag_name, + url: release.html_url, + publishedAt: release.published_at, + platform: "linux-amd64", + assetId: asset.id, + assetName: name, + sha256: asset.digest.slice(7), + }; +} +export async function verifyReleaseArchive(path: string, expected: string) { + if (!/^[a-f0-9]{64}$/.test(expected)) throw new Error("Invalid expected SHA-256"); + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) hash.update(chunk); + if (hash.digest("hex") !== expected) throw new Error("Bali release archive SHA-256 mismatch"); +} +if (import.meta.main) { + try { + const [command, input, output] = Bun.argv.slice(2); + if (command === "select" && input && output) { + const selected = selectBaliRelease(await Bun.file(input).json()); + await Bun.write(output, JSON.stringify(selected, null, 2) + "\n"); + console.log(`${selected.tag}: ${selected.assetName} (${selected.sha256})`); + } else if (command === "verify" && input && output) { + const selected = await Bun.file(input).json(); + await verifyReleaseArchive(output, selected.sha256); + console.log(`Verified ${selected.assetName}`); + } else + throw new Error( + "usage: bali-release.ts select | verify ", + ); + } catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 2; + } +} diff --git a/bin/docker.ts b/bin/docker.ts new file mode 100644 index 0000000..e8c46e0 --- /dev/null +++ b/bin/docker.ts @@ -0,0 +1,190 @@ +// Shared Docker launcher primitives for the Elide (bin/run.ts) and Bali (bin/bali-docker.ts) +// targets: process running with interrupt cleanup, user mapping, and writable-mount checks. +import { createHash, randomUUID } from "node:crypto"; +import { closeSync, openSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const RUN_LABEL_KEY = "elide.testsuite.run"; +const RUN_ID = `${Date.now()}-${process.pid}-${randomUUID()}`; +/** Label applied to every container of this launcher process, so interrupts can clean up. */ +export const RUN_LABEL = `${RUN_LABEL_KEY}=${RUN_ID}`; +const activeProcesses = new Set>(); +let handlingSignal = false; + +export function log(message: string): void { + process.stderr.write(`[bin/run] ${message}\n`); +} + +export function usageError(message: string): never { + log(message); + process.exit(2); +} + +function cleanupContainersSync(): void { + const listed = Bun.spawnSync(["docker", "ps", "-aq", "--filter", `label=${RUN_LABEL}`], { + stdout: "pipe", + stderr: "pipe", + }); + const ids = new TextDecoder().decode(listed.stdout).trim().split(/\s+/).filter(Boolean); + if (ids.length === 0) return; + log(`cleaning up ${ids.length} running container(s) for interrupted run`); + Bun.spawnSync(["docker", "rm", "-f", ...ids], { + stdout: "ignore", + stderr: "ignore", + }); +} + +function interrupt(signal: NodeJS.Signals): never { + if (handlingSignal) process.exit(130); + handlingSignal = true; + log(`received ${signal}; stopping active command and cleaning up containers`); + for (const proc of activeProcesses) { + try { + proc.kill("SIGINT"); + } catch { + // Best effort: labelled containers are forcibly removed below. + } + } + cleanupContainersSync(); + process.exit(signal === "SIGTERM" ? 143 : 130); +} + +process.on("SIGINT", () => interrupt("SIGINT")); +process.on("SIGTERM", () => interrupt("SIGTERM")); + +export async function run(args: string[], opts: { cwd?: string; env?: NodeJS.ProcessEnv } = {}): Promise { + const proc = Bun.spawn(args, { + cwd: opts.cwd ?? ROOT, + env: opts.env ?? process.env, + stdin: "inherit", + stdout: "inherit", + stderr: "inherit", + }); + activeProcesses.add(proc); + try { + return await proc.exited; + } finally { + activeProcesses.delete(proc); + } +} + +export async function runWithHeartbeat(args: string[], label: string, opts: { cwd?: string; env?: NodeJS.ProcessEnv } = {}): Promise { + log(`${label}...`); + const started = performance.now(); + const timer = setInterval(() => { + const seconds = Math.round((performance.now() - started) / 1000); + log(`${label} still running (${seconds}s)...`); + }, 5_000); + try { + const rc = await run(args, opts); + const seconds = Math.round((performance.now() - started) / 1000); + log(`${label} ${rc === 0 ? "done" : `exited ${rc}`} (${seconds}s).`); + return rc; + } finally { + clearInterval(timer); + } +} + +export async function capture(args: string[], opts: { cwd?: string; env?: NodeJS.ProcessEnv } = {}): Promise<{ + exitCode: number; + stdout: string; + stderr: string; +}> { + const proc = Bun.spawn(args, { + cwd: opts.cwd ?? ROOT, + env: opts.env ?? process.env, + stdout: "pipe", + stderr: "pipe", + }); + activeProcesses.add(proc); + try { + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + return { exitCode, stdout, stderr }; + } finally { + activeProcesses.delete(proc); + } +} + +export async function requireDocker(): Promise { + const result = await capture(["docker", "--version"]); + if (result.exitCode !== 0) usageError("docker not found on PATH"); +} + +export function platformArgs(platform: string): string[] { + return platform ? ["--platform", platform] : []; +} + +/** Run as the invoking user so bind-mounted output stays owned by them. */ +export function userArgs(uid: string, gid: string, containerPath?: string): string[] { + return [ + "--user", + `${uid}:${gid}`, + "-e", + "HOME=/work/.harness", + ...(containerPath ? ["-e", `PATH=${containerPath}`] : []), + ]; +} + +export function sha256File(path: string): string { + return createHash("sha256").update(readFileSync(path)).digest("hex"); +} + +/** Fail early with a diagnostic when a previous root-owned container left mounts unwritable. */ +export function assertWritableMountSources(dirs: string[], files: string[] = []): void { + for (const dir of dirs) { + const target = resolve(ROOT, dir, `.write-test-${process.pid}`); + try { + writeFileSync(target, ""); + unlinkSync(target); + } catch (err) { + usageError( + `${dir}/ is not writable by the current user (${err instanceof Error ? err.message : String(err)}). ` + + "Run once with --repair-ownership to fix stale root-owned files.", + ); + } + } + for (const file of files) { + try { + closeSync(openSync(resolve(ROOT, file), "a")); + } catch (err) { + usageError( + `${file} is not writable by the current user (${err instanceof Error ? err.message : String(err)}). ` + + "Run once with --repair-ownership to fix stale ownership.", + ); + } + } +} + +export async function fixHostOwnership( + image: string, + plat: string[], + uid: string, + gid: string, + label: string, + paths: string[], +): Promise { + await runWithHeartbeat( + [ + "docker", + "run", + "--rm", + "--label", + RUN_LABEL, + ...plat, + "--entrypoint", + "chown", + ...paths.flatMap((path) => ["-v", `${ROOT}/${path}:/target/${path}`]), + image, + "-R", + `${uid}:${gid}`, + ...paths.map((path) => `/target/${path}`), + ], + label, + ); +} diff --git a/bin/run.ts b/bin/run.ts index 75c5184..f49534f 100755 --- a/bin/run.ts +++ b/bin/run.ts @@ -22,6 +22,21 @@ import { availableParallelism } from "node:os"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { classifySuiteStatus } from "./suite-status"; +import { + ROOT, + RUN_LABEL, + assertWritableMountSources, + capture, + fixHostOwnership, + log, + platformArgs, + requireDocker, + run, + runWithHeartbeat, + sha256File, + usageError, + userArgs, +} from "./docker"; interface Options { elideRef: string; @@ -44,6 +59,7 @@ interface Options { interface WorkloadInfo { id: string; path?: string; + target?: string; } interface SuiteRunSummary { @@ -74,11 +90,7 @@ interface SuiteSummaryRow { changes?: SuiteChanges; } -const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const DEFAULT_ELIDE_REF = "ghcr.io/elide-dev/elide:nightly"; -const RUN_LABEL_KEY = "elide.testsuite.run"; -const RUN_ID = `${Date.now()}-${process.pid}-${randomUUID()}`; -const RUN_LABEL = `${RUN_LABEL_KEY}=${RUN_ID}`; const CONTAINER_PATH = [ "/opt/jtreg/bin", "/opt/graalvm-jdk-25.0.3/bin", @@ -90,12 +102,6 @@ const CONTAINER_PATH = [ "/sbin", "/bin", ].join(":"); -const activeProcesses = new Set>(); -let handlingSignal = false; - -function log(message: string): void { - process.stderr.write(`[bin/run] ${message}\n`); -} const COLOR = process.stderr.isTTY && !process.env.NO_COLOR; const ansi = { @@ -107,11 +113,6 @@ const ansi = { bold: (s: string) => COLOR ? `\x1b[1m${s}\x1b[0m` : s, }; -function usageError(message: string): never { - log(message); - process.exit(2); -} - function parsePositiveInt(value: string | undefined, name: string): number | undefined { if (value === undefined || value === "") return undefined; const parsed = Number.parseInt(value, 10); @@ -142,38 +143,6 @@ function planConcurrency(options: Options, suiteCount: number): ConcurrencyPlan return { cpuCount: cpus, totalBudget, suiteWorkers, threads }; } -function cleanupContainersSync(): void { - const listed = Bun.spawnSync(["docker", "ps", "-aq", "--filter", `label=${RUN_LABEL}`], { - stdout: "pipe", - stderr: "pipe", - }); - const ids = new TextDecoder().decode(listed.stdout).trim().split(/\s+/).filter(Boolean); - if (ids.length === 0) return; - log(`cleaning up ${ids.length} running container(s) for interrupted run`); - Bun.spawnSync(["docker", "rm", "-f", ...ids], { - stdout: "ignore", - stderr: "ignore", - }); -} - -function interrupt(signal: NodeJS.Signals): never { - if (handlingSignal) process.exit(130); - handlingSignal = true; - log(`received ${signal}; stopping active command and cleaning up containers`); - for (const proc of activeProcesses) { - try { - proc.kill("SIGINT"); - } catch { - // Best effort: labelled containers are forcibly removed below. - } - } - cleanupContainersSync(); - process.exit(signal === "SIGTERM" ? 143 : 130); -} - -process.on("SIGINT", () => interrupt("SIGINT")); -process.on("SIGTERM", () => interrupt("SIGTERM")); - function parseArgs(argv: string[]): Options { const options: Options = { elideRef: DEFAULT_ELIDE_REF, @@ -262,76 +231,6 @@ function parseArgs(argv: string[]): Options { return options; } -function platformArgs(platform: string): string[] { - return platform ? ["--platform", platform] : []; -} - -function userArgs(uid: string, gid: string): string[] { - return ["--user", `${uid}:${gid}`, "-e", "HOME=/work/.harness", "-e", `PATH=${CONTAINER_PATH}`]; -} - -async function run(args: string[], opts: { cwd?: string; env?: NodeJS.ProcessEnv } = {}): Promise { - const proc = Bun.spawn(args, { - cwd: opts.cwd ?? ROOT, - env: opts.env ?? process.env, - stdin: "inherit", - stdout: "inherit", - stderr: "inherit", - }); - activeProcesses.add(proc); - try { - return await proc.exited; - } finally { - activeProcesses.delete(proc); - } -} - -async function runWithHeartbeat(args: string[], label: string, opts: { cwd?: string; env?: NodeJS.ProcessEnv } = {}): Promise { - log(`${label}...`); - const started = performance.now(); - const timer = setInterval(() => { - const seconds = Math.round((performance.now() - started) / 1000); - log(`${label} still running (${seconds}s)...`); - }, 5_000); - try { - const rc = await run(args, opts); - const seconds = Math.round((performance.now() - started) / 1000); - log(`${label} ${rc === 0 ? "done" : `exited ${rc}`} (${seconds}s).`); - return rc; - } finally { - clearInterval(timer); - } -} - -async function capture(args: string[], opts: { cwd?: string; env?: NodeJS.ProcessEnv } = {}): Promise<{ - exitCode: number; - stdout: string; - stderr: string; -}> { - const proc = Bun.spawn(args, { - cwd: opts.cwd ?? ROOT, - env: opts.env ?? process.env, - stdout: "pipe", - stderr: "pipe", - }); - activeProcesses.add(proc); - try { - const [stdout, stderr, exitCode] = await Promise.all([ - new Response(proc.stdout).text(), - new Response(proc.stderr).text(), - proc.exited, - ]); - return { exitCode, stdout, stderr }; - } finally { - activeProcesses.delete(proc); - } -} - -async function requireDocker(): Promise { - const result = await capture(["docker", "--version"]); - if (result.exitCode !== 0) usageError("docker not found on PATH"); -} - function dockerImageName(elideRef: string): string { return `elide-harness:${elideRef.replaceAll(/[/:@]/g, "_")}`; } @@ -340,10 +239,6 @@ function isLocalInstallDir(path: string): boolean { return existsSync(path) && statSync(path).isDirectory() && existsSync(resolve(path, "bin/elide")); } -function sha256File(path: string): string { - return createHash("sha256").update(readFileSync(path)).digest("hex"); -} - async function buildHarnessImage(options: Options, image: string, plat: string[]): Promise { const { elideRef } = options; if (isLocalInstallDir(elideRef)) { @@ -436,6 +331,8 @@ function parseRegistry(registryPath: string): WorkloadInfo[] { if (id) current.id = id; const path = line.match(/^\s*path\s*=\s*"([^"]+)"/)?.[1]; if (path) current.path = path; + const target = line.match(/^\s*target\s*=\s*"([^"]+)"/)?.[1]; + if (target) current.target = target; } if (current?.id) workloads.push(current); return workloads; @@ -449,58 +346,6 @@ async function suiteVersion(workload: WorkloadInfo): Promise { return result.exitCode === 0 ? result.stdout.trim() : "unknown"; } -async function fixHostOwnership(image: string, plat: string[], uid: string, gid: string, label: string): Promise { - await runWithHeartbeat([ - "docker", - "run", - "--rm", - "--label", - RUN_LABEL, - ...plat, - "--entrypoint", - "chown", - "-v", - `${ROOT}/reports:/target/reports`, - "-v", - `${ROOT}/expectations:/target/expectations`, - "-v", - `${ROOT}/.harness:/target/.harness`, - "-v", - `${ROOT}/README.md:/target/README.md`, - image, - "-R", - `${uid}:${gid}`, - "/target/reports", - "/target/expectations", - "/target/.harness", - "/target/README.md", - ], label); -} - -function assertWritableMountSources(): void { - for (const dir of ["reports", "expectations", ".harness"]) { - const target = resolve(ROOT, dir, `.write-test-${process.pid}`); - try { - writeFileSync(target, ""); - unlinkSync(target); - } catch (err) { - usageError( - `${dir}/ is not writable by the current user (${err instanceof Error ? err.message : String(err)}). ` + - "Run once with --repair-ownership to fix stale root-owned files.", - ); - } - } - - try { - closeSync(openSync(resolve(ROOT, "README.md"), "a")); - } catch (err) { - usageError( - `README.md is not writable by the current user (${err instanceof Error ? err.message : String(err)}). ` + - "Run once with --repair-ownership to fix stale ownership.", - ); - } -} - function walkSummaryPaths(root: string): string[] { if (!existsSync(root)) return []; const out: string[] = []; @@ -699,18 +544,23 @@ function renderFinalSuiteSummary(rows: SuiteSummaryRow[]): void { process.stderr.write(`${bottom}\n\n`); } -async function main(): Promise { +async function main(argv = Bun.argv.slice(2)): Promise { process.chdir(ROOT); - const options = parseArgs(Bun.argv.slice(2)); + const options = parseArgs(argv); const plat = platformArgs(options.platform); const hostUid = (await $`id -u`.text()).trim(); const hostGid = (await $`id -g`.text()).trim(); - const user = userArgs(hostUid, hostGid); + const user = userArgs(hostUid, hostGid, CONTAINER_PATH); const registryPath = resolve(ROOT, "registry.toml"); const workloads = parseRegistry(registryPath); + // Workloads for other runtimes (registry `target`) are run through `--target `. const suites = options.allSuites && options.suites.length === 0 - ? workloads.map((workload) => workload.id) + ? workloads.filter((workload) => (workload.target ?? "elide") === "elide").map((workload) => workload.id) : options.suites.length ? options.suites : ["test262"]; + for (const suite of suites) { + const target = workloads.find((workload) => workload.id === suite)?.target; + if (target && target !== "elide") usageError(`suite '${suite}' targets ${target}; run it with --target ${target}`); + } if (options.prepareSuites) { const rc = await runWithHeartbeat( @@ -741,9 +591,14 @@ async function main(): Promise { if (!existsSync(resolve(ROOT, "README.md"))) closeSync(openSync(resolve(ROOT, "README.md"), "a")); if (options.repairOwnership) { - await fixHostOwnership(image, plat, hostUid, hostGid, "repairing writable mount ownership before suite runs"); + await fixHostOwnership(image, plat, hostUid, hostGid, "repairing writable mount ownership before suite runs", [ + "reports", + "expectations", + ".harness", + "README.md", + ]); } else { - assertWritableMountSources(); + assertWritableMountSources(["reports", "expectations", ".harness"], ["README.md"]); } const expMode = options.ratchet ? "rw" : "ro"; @@ -852,7 +707,33 @@ async function main(): Promise { } try { - process.exit(await main()); + const argv = Bun.argv.slice(2); + const targetIndex = argv.indexOf("--target"); + const target = targetIndex < 0 ? "elide" : argv[targetIndex + 1]; + if (targetIndex >= 0) argv.splice(targetIndex, 2); + if (target === "bali") { + const executionIndex = argv.indexOf("--execution"); + const execution = executionIndex < 0 ? "docker" : argv[executionIndex + 1]; + if (executionIndex >= 0) argv.splice(executionIndex, 2); + const baliSuites = parseRegistry(resolve(ROOT, "registry.toml")) + .filter((workload) => workload.target === "bali") + .map((workload) => workload.id); + if (argv.includes("--help")) { + console.log( + `usage: bun run testsuite --target bali --bali-home [--ratchet] [--suite ${baliSuites.join("|")}] [--execution docker|native]\n` + + "Docker (default) runs linux/amd64 with the reference JDK fixed in the image. --execution native runs on this host, " + + "using JAVA_HOME or --reference-home as the reference JDK. Results go to reports/bali/ and are checked against expectations/.toml like Elide suites.", + ); + } else if (execution === "docker") { + const { runBaliDocker } = await import("./bali-docker"); + process.exitCode = await runBaliDocker(argv, ROOT, RUN_LABEL, run, baliSuites); + } else if (execution === "native") { + const { runBaliNative } = await import("./bali-docker"); + process.exitCode = await runBaliNative(argv, ROOT, baliSuites); + } else throw new Error(`Unknown execution environment: ${execution}`); + } else if (target === "elide") { + process.exit(await main(argv)); + } else throw new Error(`Unknown target: ${target}`); } catch (err) { log(`ERROR: unexpected failure: ${err instanceof Error ? (err.stack ?? err.message) : String(err)}`); process.exit(2); diff --git a/docker/bali.Dockerfile b/docker/bali.Dockerfile new file mode 100644 index 0000000..15f1837 --- /dev/null +++ b/docker/bali.Dockerfile @@ -0,0 +1,22 @@ +# syntax=docker/dockerfile:1 +# Match Elide's Linux/Debian harness environment; stock Java is Bali's oracle. +FROM eclipse-temurin:25.0.2_10-jdk@sha256:1bda4d9e668f44f399abed30636c34e0befb727408fba27b1e6aaefcf9df346b AS reference +FROM node:25-trixie@sha256:b6cf8d20ee78aa10f0a6b98242e26d547869801abd6bc3f7094b0a54301dee02 +RUN npm install -g bun@1.4.0 \ + && apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl git unzip \ + libzstd1 libfreetype6 fontconfig libx11-6 libxext6 libxi6 libxrender1 libxtst6 libasound2t64 \ + && rm -rf /var/lib/apt/lists/* +COPY --from=reference /opt/java/openjdk /opt/reference-jdk +ENV JAVA_HOME=/opt/reference-jdk +ENV PATH=/opt/reference-jdk/bin:$PATH +# The shared harness (cli.ts, expectations, ratchet, reports) needs its dependencies; +# the Elide-host postinstall is irrelevant for Bali and is skipped. +WORKDIR /work/harness +COPY harness/package.json harness/bun.lock ./ +RUN bun install --frozen-lockfile --ignore-scripts +COPY harness/src ./src +WORKDIR /work +# registry.toml, manifests/, expectations/, reports/, and .harness/ are bind-mounted at +# run time, exactly as for the Elide harness container. +ENTRYPOINT ["bun", "/work/harness/src/cli.ts"] diff --git a/expectations/jdk-jtreg.toml b/expectations/jdk-jtreg.toml new file mode 100644 index 0000000..159b939 --- /dev/null +++ b/expectations/jdk-jtreg.toml @@ -0,0 +1,12 @@ +# Bali jdk-jtreg expectations. Keys are picomatch globs over test/jdk file paths +# such as java/lang/**. Same format and semantics as the Elide suites: +# [skip] mutes files (not scored), [fail] records known Bali-only failures. +# Failures observed by a `--ratchet` run are recorded automatically in +# jdk-jtreg.ratchet.toml; hand-curated entries with reasons belong here. +# +# Adapter gaps (unsupported jtreg directives) and reference-JDK problems are +# reported as skips by the adapter itself and need no entries. + +[skip] + +[fail] diff --git a/harness/src/adapters/index.ts b/harness/src/adapters/index.ts index 9ab292c..91c430d 100644 --- a/harness/src/adapters/index.ts +++ b/harness/src/adapters/index.ts @@ -1,6 +1,7 @@ import type { Adapter } from "./types"; import { cpythonCoreAdapter } from "./cpython-core"; import { javacJtregAdapter } from "./javac-jtreg"; +import { jdkJtregAdapter } from "./jdk-jtreg"; import { nodeApiAdapter } from "./node-api"; import { test262Adapter } from "./test262"; import { wptWintertcAdapter } from "./wpt-wintertc"; @@ -8,6 +9,7 @@ import { wptWintertcAdapter } from "./wpt-wintertc"; export const ADAPTERS: Record = { "cpython-core": cpythonCoreAdapter, "javac-jtreg": javacJtregAdapter, + "jdk-jtreg": jdkJtregAdapter, "node-api": nodeApiAdapter, test262: test262Adapter, "wpt-wintertc": wptWintertcAdapter, diff --git a/harness/src/adapters/javac-jtreg.ts b/harness/src/adapters/javac-jtreg.ts index 8b52711..10c12c7 100644 --- a/harness/src/adapters/javac-jtreg.ts +++ b/harness/src/adapters/javac-jtreg.ts @@ -1,17 +1,11 @@ -import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import { chmodSync, cpSync, existsSync, mkdirSync, readdirSync, readFileSync, statSync, symlinkSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import picomatch from "picomatch"; import type { Adapter, AdapterContext } from "./types"; import type { TestResult } from "../results/schema"; import { loadManifest } from "../manifest"; import { runProcess, type ProcessRunResult } from "./process"; - -const STATUS: Record = { - passed: "pass", - failed: "fail", - error: "error", - "not run": "skip", -}; +import { createJtregRunRoot, isJtregRunnerExit, jtregCommonArgs, jtregOutcome } from "./jtreg"; const JTREG_DEFAULT_ACTION_TIMEOUT_SECONDS = 120; @@ -207,8 +201,7 @@ export function buildWrapperJdk(wrapperJdk: string, realJdkHome: string, repoRoo } export async function createJtregRunLayout(ctx: AdapterContext, realJdkHome?: string): Promise { - mkdirSync(ctx.workspacePath, { recursive: true }); - const runRoot = mkdtempSync(join(ctx.workspacePath, "jtreg-run-")); + const runRoot = createJtregRunRoot(ctx.workspacePath); const workDir = join(runRoot, "JTwork"); const reportDir = join(runRoot, "JTreport"); const wrapperJdk = join(runRoot, "wrapper-jdk"); @@ -276,7 +269,7 @@ export function parseJtregSummary(text: string): TestResult[] { const path = statusFirst?.[2] ?? pathFirst?.[1]; const message = pathFirst?.[3]?.trim(); if (!statusName || !path) continue; - const status = message?.startsWith("Test ignored:") ? "skip" : STATUS[statusName.toLowerCase()]; + const status = message?.startsWith("Test ignored:") ? "skip" : jtregOutcome(statusName); if (!status) continue; results.push({ kind: "test", @@ -322,26 +315,21 @@ function runnerFailureResult(result: ProcessRunResult): TestResult { } function isJtregRunnerFailure(result: ProcessRunResult): boolean { - if (result.timedOut) return true; - // jtreg uses 1 for no tests, 2 for failed tests, and 3 for errored tests. - // When a parseable summary exists, those outcomes are carried per test. - return result.exitCode !== 0 && result.exitCode !== 1 && result.exitCode !== 2 && result.exitCode !== 3; + // When a parseable summary exists, jtreg's 1/2/3 exit codes are carried per test. + return result.timedOut || isJtregRunnerExit(result.exitCode); } function appendCapped(output: string, text: string, cap: number): string { return output.length < cap ? output + text.slice(0, cap - output.length) : output; } -function jtregTimeoutArgs(settings: Record): string[] { +function jtregTimeoutFactor(settings: Record): number | undefined { const caseTimeoutSeconds = Number(settings.jtregCaseTimeoutSeconds); if (Number.isFinite(caseTimeoutSeconds) && caseTimeoutSeconds > 0) { - return [`-timeoutFactor:${caseTimeoutSeconds / JTREG_DEFAULT_ACTION_TIMEOUT_SECONDS}`]; + return caseTimeoutSeconds / JTREG_DEFAULT_ACTION_TIMEOUT_SECONDS; } - const timeoutFactor = Number(settings.jtregTimeoutFactor ?? 1); - return Number.isFinite(timeoutFactor) && timeoutFactor > 0 - ? [`-timeoutFactor:${timeoutFactor}`] - : []; + return Number.isFinite(timeoutFactor) && timeoutFactor > 0 ? timeoutFactor : undefined; } async function* readStreamLines( @@ -402,13 +390,9 @@ export async function* runJavacJtreg(ctx: AdapterContext): AsyncIterable join(jtregLangtoolsRoot, test)), ]; const emitted = new Set(); diff --git a/harness/src/adapters/jdk-jtreg.test.ts b/harness/src/adapters/jdk-jtreg.test.ts new file mode 100644 index 0000000..fa4788e --- /dev/null +++ b/harness/src/adapters/jdk-jtreg.test.ts @@ -0,0 +1,320 @@ +import { expect, test } from "bun:test"; +import { chmod, mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + cleanEnv, + MEASUREMENT_ENVIRONMENT, + discover, + flatten, + jdkJtregAdapter, + hasTest, + incomplete, + markdown, + parseJtr, + portableReason, + propertiesReason, + runSuiteOnRuntime, + runtimeProgress, + summary, + unflatten, + type Report, + type Status, +} from "./jdk-jtreg"; +import { compare as compareExpectations } from "../expectations/compare"; +import { parseExpectations, type Expectations } from "../expectations/load"; +import { ratchetCandidates } from "../expectations/ratchet"; +import manifest from "../../../manifests/bali-jdk.json"; + +const none = (): Expectations => ({ entries: [], ratchet: new Set() }); +// The harness exits 2 for an incomplete run (the adapter throws), else 1 iff regressions. +const code = (r: Report) => + incomplete(r) ? 2 : compareExpectations(flatten(r), none()).regressions.length ? 1 : 0; + +const jtr = (status: string) => + `#Test Results (version 2)\n#-----testresult-----\nexecStatus=${status}\n#section:main\nexecStatus=Passed. Fake output\n`; +const report = (statuses: Status[]): Report => ({ + schema: 2, + suite: "fixture", + createdAt: "2026-09-07T00:00:00Z", + metadata: {}, + exitCodes: { + reference: 0, + bali: statuses.every((status) => status === "pass" || status === "unsupported") ? 0 : 2, + }, + tests: statuses.map((status, index) => ({ + id: `java/lang/${index}.java`, + area: "java/lang", + unsupported: status === "unsupported" ? "Unsupported @library directive" : null, + reference: { status: status === "unsupported" ? "unsupported" : "pass", detail: "reference" }, + bali: { status, detail: status }, + })), +}); +const main = "public static void main(String[] args) {}"; +const source = `/* @test\n * @run main/othervm Example\n */\n${main}`; + +test("jtr parsing uses the result header, never guest output or missing results", () => { + expect(parseJtr(jtr("Passed. Execution successful")).status).toBe("pass"); + expect(parseJtr(jtr("Passed. Skipped: Windows only")).status).toBe("skipped"); + expect(parseJtr(jtr("Failed. main threw exception")).status).toBe("fail"); + expect(parseJtr(jtr("Error. Cannot compile")).status).toBe("error"); + expect(parseJtr(jtr("Error. Program timed out")).status).toBe("timeout"); + expect(parseJtr(jtr("Not run. Filtered")).status).toBe("blocked"); + expect(parseJtr(undefined).status).toBe("blocked"); + expect(parseJtr("#section:main\nexecStatus=Passed. Fake output").status).toBe("blocked"); +}); + +test("discovery parses test descriptions without mistaking ordinary javadoc for jtreg tags", () => { + expect(hasTest(source)).toBe(true); + expect(hasTest("# @test\n# shell test")).toBe(true); + expect(hasTest("class Fixture {} ")).toBe(false); + expect( + portableReason("java/lang/Example.java", source + "\n/**\n * @throws Exception reason\n */"), + ).toBeNull(); + for (const tag of [ + "requires vm.gc.G1", + "library /test/lib", + "modules java.base/jdk.internal.misc", + "ignore broken", + "build Helper", + "run main -XX:+WhiteBoxAPI Example", + "run testng Example", + "test id=second", + ]) { + expect( + portableReason("java/lang/Example.java", `/* @test\n * @${tag}\n */\n${main}`), + ).not.toBeNull(); + } + expect(propertiesReason("# comment\nallowSmartActionArgs=true\n")).toBeNull(); + expect(propertiesReason("modules=java.desktop\n")).not.toBeNull(); + expect( + portableReason("java/lang/Example.java", `/* @test\n @library /test/lib\n */\n${main}`), + ).toBe("Unsupported @library directive"); + expect(hasTest(`/*\n @test\n */`)).toBe(true); + expect( + portableReason( + "java/lang/Example.java", + `/* @test\n * @run main Example\n * extraArgument\n */\n${main}`, + ), + ).not.toBeNull(); +}); + +test("inventory retains unsupported tests, excludes support files, and respects inherited settings", async () => { + const root = await mkdtemp(join(tmpdir(), "bali-compat-")); + try { + await mkdir(join(root, "java/lang/restricted"), { recursive: true }); + await Bun.write(join(root, "java/lang/Example.java"), source); + await Bun.write(join(root, "java/lang/Fixture.java"), "class Fixture {}"); + await Bun.write(join(root, "java/lang/Script.sh"), "# @test\n# @run shell Script.sh\n"); + await Bun.write(join(root, "java/lang/restricted/Example.java"), source); + await Bun.write(join(root, "java/lang/restricted/TEST.properties"), "modules=java.desktop\n"); + const rows = await discover(root); + expect(rows.length).toBe(3); + expect(rows[0]!.unsupported).toBeNull(); + expect(rows[1]!.unsupported).toBe("Non-Java test action"); + expect(rows[2]!.unsupported).toContain("TEST.properties"); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("manifest declares a pinned inventory scope rather than a handpicked passing list", () => { + expect(manifest.scope).toBe("test/jdk"); + expect(manifest).not.toHaveProperty("tests"); + expect(manifest.source.revision).toMatch(/^[a-f0-9]{40}$/); + for (const pin of [manifest.source, manifest.jtreg]) expect(pin.sha256).toMatch(/^[a-f0-9]{64}$/); +}); + +test("only harness aborts or missing results are incomplete; failures are ordinary results", () => { + expect(code(report(["pass", "unsupported"]))).toBe(0); + expect(code(report(["fail"]))).toBe(1); + expect(code(report(["timeout"]))).toBe(1); + expect(code(report(["blocked"]))).toBe(2); + expect(code(report([]))).toBe(2); + const referenceIssue = report(["pass"]); + referenceIssue.tests[0]!.reference.status = "fail"; + referenceIssue.exitCodes.reference = 2; + // A reference problem is not a Bali regression: it is skipped, not scored. + expect(code(referenceIssue)).toBe(0); + expect(incomplete(referenceIssue)).toBe(false); + expect(summary(referenceIssue).verifiedPassing).toBe(0); + expect(summary(referenceIssue).referenceIssues).toBe(1); + const badHarness = report(["pass"]); + for (const exit of [1, 4, 5, 6]) { + badHarness.exitCodes.bali = exit; + expect(incomplete(badHarness)).toBe(true); + expect(code(badHarness)).toBe(2); + } + const testError = report(["error"]); + testError.exitCodes.bali = 3; + expect(code(testError)).toBe(1); +}); + +test("flattening keeps both-pass as pass, Bali-only failures as failures, and other outcomes as skips", () => { + const flat = flatten(report(["pass", "fail", "timeout", "skipped", "unsupported"])); + expect(flat.map((t) => t.status)).toEqual(["pass", "fail", "error", "fail", "skip"]); + expect(flat[1]!.meta).toMatchObject({ area: "java/lang", bali: { status: "fail" } }); + expect(flat[4]!.message).toBe("Unsupported @library directive"); + const referenceIssue = report(["pass"]); + referenceIssue.tests[0]!.reference.status = "fail"; + expect(flatten(referenceIssue)[0]).toMatchObject({ + status: "skip", + message: "Reference fail: reference", + }); + // Reclassifying a passing file as unsupported loses its pass rather than hiding it. + const reclassified = report(["pass"]); + reclassified.tests[0]!.unsupported = "Unsupported directive"; + expect(flatten(reclassified)[0]!.status).toBe("skip"); + expect(summary(reclassified).verifiedPassing).toBe(0); +}); + +test("expectations and ratchet apply to flattened results exactly like Elide suites", () => { + const tests = flatten(report(["pass", "fail", "error"])); + const exp = parseExpectations('[fail]\n"java/lang/1.java" = "known"\n'); + expect(compareExpectations(tests, exp).regressions.map((t) => t.id)).toEqual(["java/lang/2.java"]); + expect(ratchetCandidates(tests, exp)).toEqual(["java/lang/2.java"]); + exp.ratchet.add("java/lang/2.java"); + expect(compareExpectations(tests, exp).regressions).toEqual([]); + const fixed = compareExpectations(flatten(report(["pass", "pass", "pass"])), exp); + expect(fixed.newPasses.map((t) => t.id)).toEqual(["java/lang/1.java", "java/lang/2.java"]); + const muted = parseExpectations('[skip]\n"java/lang/**" = "muted"\n'); + expect(compareExpectations(tests, muted).counts).toMatchObject({ skip: 3, pass: 0 }); +}); + +test("summary exposes reference problems and adapter gaps in the fixed inventory", () => { + const data = report(["pass", "unsupported", "fail"]); + expect(summary(data)).toEqual({ + inventory: 3, + runnable: 2, + unsupported: 1, + verifiedPassing: 1, + baliFailures: 1, + referenceIssues: 0, + incomplete: false, + }); + expect(markdown(data)).toContain("3 test files inventoried"); + expect(markdown(data)).toContain("not Java SE certification"); +}); + +test("measurement environment ignores terminal locale and uses fixed settings", async () => { + const ambient = { + LANG: "cs_CZ.UTF-8", + LC_ALL: "", + LC_CTYPE: "UTF-8", + TZ: "Pacific/Honolulu", + PATH: "/usr/bin", + JAVA_TOOL_OPTIONS: "-Xmx1g", + }; + const env = cleanEnv("/jdk", ambient); + expect(env.LANG).toBe("C.UTF-8"); + expect(env.LC_ALL).toBe("C.UTF-8"); + expect(env.LC_CTYPE).toBe("C.UTF-8"); + expect(env.TZ).toBeUndefined(); + expect(env.JAVA_TOOL_OPTIONS).toBeUndefined(); + expect(env.JAVA_HOME).toBe("/jdk"); + expect(env.PATH).toBe("/jdk/bin:/usr/bin"); + expect(ambient.TZ).toBe("Pacific/Honolulu"); + expect(MEASUREMENT_ENVIRONMENT).toEqual({ + LANG: "C.UTF-8", + LC_ALL: "C.UTF-8", + LC_CTYPE: "C.UTF-8", + TZ: null, + }); +}); + +test("runtime progress counts only completed results, including timeouts and skips", async () => { + const work = await mkdtemp(join(tmpdir(), "bali-compat-progress-")); + try { + await Bun.write(join(work, "Pass.jtr"), jtr("Passed. Execution successful")); + await Bun.write(join(work, "Timeout.jtr"), jtr("Error. Program timed out")); + await Bun.write(join(work, "Skip.jtr"), jtr("Passed. Skipped: unsupported platform")); + await Bun.write(join(work, "Running.jtr"), "#Test Results (version 2)\n"); + expect( + await runtimeProgress(work, [ + "Pass.java", + "Timeout.java", + "Skip.java", + "Running.java", + "Missing.java", + ]), + ).toEqual({ completed: 3, passed: 1, failed: 1, skipped: 1 }); + } finally { + await rm(work, { recursive: true, force: true }); + } +}); + +test("native adapter invokes jtreg with selected runtime and stock compiler and reads fresh results", async () => { + const root = await mkdtemp(join(tmpdir(), "bali-native-adapter-")); + try { + const ref = join(root, "reference-jdk"); + await mkdir(join(ref, "bin"), { recursive: true }); + const java = join(ref, "bin/java"); + await Bun.write( + java, + `#!/bin/sh +exec '${process.execPath.replaceAll("'", "'\\''")}' '${join(root, "fake-jtreg.ts")}' "$@" +`, + ); + await chmod(java, 0o755); + await Bun.write( + join(root, "fake-jtreg.ts"), + ` +const args = process.argv.slice(2); +const work = args.find(a => a.startsWith("-w:"))!.slice(3); +await Bun.write(work + "/Example.jtr", "#-----testresult-----\\nexecStatus=Passed. Execution successful\\n#section:main\\n"); +console.log("fixture jtreg started"); +console.error("fixture jtreg diagnostic"); +while (!(await Bun.file(${JSON.stringify(join(root, "release"))}).exists())) await Bun.sleep(10); +console.log("fixture jtreg completed"); +`, + ); + const home = join(root, "bali-jdk"); + const output = join(root, "bali"); + const running = runSuiteOnRuntime( + output, + join(root, "suite"), + "/fixture/jtreg.jar", + ref, + home, + [{ id: "Example.java", area: "fixture", unsupported: null }], + manifest.execution, + ); + try { + const deadline = Date.now() + 3000; + let live = ""; + while (Date.now() < deadline) { + const file = Bun.file(join(output, "harness.log")); + if (await file.exists()) live = await file.text(); + if (live.includes("fixture jtreg diagnostic")) break; + await Bun.sleep(10); + } + expect(live).toContain("fixture jtreg started"); + expect(live).toContain("fixture jtreg diagnostic"); + expect(live).not.toContain("fixture jtreg completed"); + } finally { + await Bun.write(join(root, "release"), "release"); + await running; + } + const result = await running; + expect(result.exitCode).toBe(0); + expect(result.results.get("Example.java")?.status).toBe("pass"); + const args = await Bun.file(join(output, "command.json")).json(); + expect(args).toContain("-testjdk:" + home); + expect(args).toContain("-compilejdk:" + ref); + expect(args).toContain("-javaoption:-Xmx2g"); + expect(await Bun.file(join(output, "harness.log")).text()).toContain("fixture jtreg completed"); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("flattened results rebuild the paired report for the differential files", async () => { + const original = report(["pass", "fail", "timeout", "unsupported"]); + const rows = unflatten(flatten(original)); + expect(rows).toEqual(original.tests); + const files = await jdkJtregAdapter.reports!({} as never, flatten(original)); + expect(Object.keys(files).sort()).toEqual(["coverage.svg", "differential.md"]); + expect(files["differential.md"]).toContain("4 test files inventoried"); + expect(files["differential.md"]).toContain("results.json.gz"); + expect(files["coverage.svg"]).toContain("3/4 runnable; 1 verified passes"); +}); diff --git a/harness/src/adapters/jdk-jtreg.ts b/harness/src/adapters/jdk-jtreg.ts new file mode 100644 index 0000000..9f4b576 --- /dev/null +++ b/harness/src/adapters/jdk-jtreg.ts @@ -0,0 +1,624 @@ +/** + * RFC-0016: discover the pinned OpenJDK test/jdk inventory and measure runtime support. + * + * The original TEST.ROOT probes HotSpot WhiteBox/diagnostic flags even for + * portable cases. Both runtimes use the same portable root; unsupported test + * directives remain visible as adapter gaps instead of disappearing from the inventory. + * All sibling fixtures are preserved. Stock javac compiles both runtime selections. + * + * Archives are verified on every use and each run gets fresh work directories. + * Reports distinguish verified passes, Bali gaps, reference issues, and unsupported + * files. Paired outcomes are flattened into Elide's single-status model, so the shared + * expectations, ratchet, and report code protect previous passes without requiring all + * known gaps to be fixed first. + * Local binaries are identified by digest, never attributed to the current + * source commit. No network or process work happens when imported by tests. + */ +import { $ } from "bun"; +import { createHash } from "node:crypto"; +import { closeSync, createReadStream, openSync } from "node:fs"; +import { mkdir, realpath, rename, rm } from "node:fs/promises"; +import { arch, release } from "node:os"; +import { basename, dirname, join, resolve } from "node:path"; +import type { Adapter, AdapterContext } from "./types"; +import { createJtregRunRoot, isJtregTimeout, jtregCommonArgs } from "./jtreg"; +import type { Result as HarnessResult, TestResult } from "../results/schema"; +export const REPO = resolve(import.meta.dir, "../../.."); +const HEAP_CAP = "-Xmx2g"; + +// Bump the protocol only when result meaning changes, not on ordinary refactors. +export const PROTOCOL = 2; +export const ROOT = "requiredVersion=7.5.1+1\nuseNewOptions=true\nuseNewPatchModule=true\n"; +/** manifests/bali-jdk.json: pinned corpus, jtreg build, and fixed execution options. */ +export type Manifest = { + id: string; + scope: string; + source: { url: string; sha256: string; revision: string }; + jtreg: { url: string; sha256: string }; + execution: { concurrency: number; timeoutFactor: number; headless: boolean }; +}; +const ARTIFACTS = ["bin/java", "bin/bali", "lib/modules", "release"]; +export type Status = "pass" | "fail" | "error" | "timeout" | "blocked" | "unsupported" | "skipped"; +export type Result = { status: Status; detail: string }; +export type Entry = { id: string; area: string; unsupported: string | null }; +export type Row = Entry & { reference: Result; bali: Result }; +export type Report = { + schema: 2; + suite: string; + createdAt: string; + metadata: Record; + exitCodes: { reference: number; bali: number }; + tests: Row[]; +}; +const CACHE = resolve(process.env.BALI_JTREG_CACHE ?? join(REPO, ".harness/bali/archives")); + +const digest = (value: string) => createHash("sha256").update(value).digest("hex"); +async function fileDigest(path: string): Promise { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) hash.update(chunk); + return hash.digest("hex"); +} + +/** Missing/malformed results never become passes; only the result header counts. */ +export function parseJtr(text: string | undefined): Result { + if (text === undefined) return { status: "blocked", detail: "No jtreg result; see harness.log" }; + const header = text.split("#-----testresult-----\n")[1]?.split("\n#section:")[0]; + const detail = header?.match(/^execStatus=(.*)$/m)?.[1]; + if (!detail) return { status: "blocked", detail: "Missing execStatus in jtreg result" }; + if (detail.startsWith("Passed. Skipped")) return { status: "skipped", detail }; + if (detail.startsWith("Passed.")) return { status: "pass", detail }; + if (/^Failed\.|^Error\./.test(detail) && isJtregTimeout(detail)) + return { status: "timeout", detail }; + if (detail.startsWith("Failed.")) return { status: "fail", detail }; + if (detail.startsWith("Error.")) return { status: "error", detail }; + return { status: "blocked", detail }; +} + +/** Test-bearing files are counted once; multiple @test variants remain visible but unsupported. */ +export function hasTest(source: string): boolean { + return /(?:\/\*|^[ \t]*(?:\*|#)?|