From 39456dee4f2e29a4c91533d1dd3a154ee1f00589 Mon Sep 17 00:00:00 2001 From: Artur Stolear Date: Tue, 20 Oct 2026 01:05:53 +0200 Subject: [PATCH 1/2] build: add controlled net10 and net11 runtime comparison --- .github/workflows/runtime-comparison.yml | 47 +++++++ build/runtime-comparison.md | 52 ++++++++ build/runtime-comparison.mjs | 148 +++++++++++++++++++++++ 3 files changed, 247 insertions(+) create mode 100644 .github/workflows/runtime-comparison.yml create mode 100644 build/runtime-comparison.md create mode 100644 build/runtime-comparison.mjs diff --git a/.github/workflows/runtime-comparison.yml b/.github/workflows/runtime-comparison.yml new file mode 100644 index 0000000000..eb322fca53 --- /dev/null +++ b/.github/workflows/runtime-comparison.yml @@ -0,0 +1,47 @@ +name: Runtime comparison + +on: + workflow_dispatch: + pull_request: + branches: [main] + paths: + - build/runtime-comparison.mjs + - build/runtime-comparison.md + - .github/workflows/runtime-comparison.yml + +permissions: + contents: read + +jobs: + compare: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + DOTNET_CLI_TELEMETRY_OPTOUT: '1' + DOTNET_NOLOGO: '1' + steps: + - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + global-json-file: global.json + dotnet-version: '10.0.x' + - name: Publish both frameworks from the same source + run: | + dotnet publish src/GitVersion.App -c Release -f net10.0 --no-self-contained -o artifacts/runtime-app/net10 + dotnet publish src/GitVersion.App -c Release -f net11.0 --no-self-contained -o artifacts/runtime-app/net11 + - name: Compare runtimes and verify output parity + run: | + runtime10=$(dotnet --list-runtimes | awk '$1 == "Microsoft.NETCore.App" && $2 ~ /^10\./ {print $2}' | sort -V | tail -1) + runtime11=$(dotnet --list-runtimes | awk '$1 == "Microsoft.NETCore.App" && $2 ~ /^11\./ {print $2}' | sort -V | tail -1) + node build/runtime-comparison.mjs artifacts/runtime-app/net10/gitversion.dll artifacts/runtime-app/net11/gitversion.dll "$runtime10" "$runtime11" artifacts/runtime-comparison + cat artifacts/runtime-comparison/summary.md >> "$GITHUB_STEP_SUMMARY" + - name: Upload raw measurements and runtime evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: runtime-comparison + path: artifacts/runtime-comparison + if-no-files-found: error diff --git a/build/runtime-comparison.md b/build/runtime-comparison.md new file mode 100644 index 0000000000..86d7dc6cd5 --- /dev/null +++ b/build/runtime-comparison.md @@ -0,0 +1,52 @@ +# .NET runtime comparison + +This optional Linux experiment supports #5210. It does not impose a timing gate +or establish a GitVersion speedup. Run the **Runtime comparison** workflow manually; +pull requests changing the harness also run it. Both frameworks are published +with the SDK pinned in `global.json`, from the same source on one runner. + +The three variants separate runtime-only effects (the identical net10 publish +directory on runtimes 10 and 11) from framework/build differences (net11 on 11). +Each run selects exact installed runtime versions with `dotnet exec --fx-version` +and disables roll-forward. Untimed host traces verify the selected CoreCLR path. +Environment metadata records source SHA, dirty state, SDK, runtimes, Git and CPU. +Child processes receive a minimal environment so CI branch/agent variables, +Git configuration overrides and inherited host tracing cannot alter the fixtures. +The workflow resolves installed patch versions once and records them; comparisons +across workflow runs must account for changing patches and runner hardware. + +Two reproducible packed repositories contain 100 and 5,000 linear commits, fixed +timestamps and a version tag on the first commit. They are deliberately bounded fixtures, not a model +of every branching strategy or repository. Head IDs are retained with results. +Every case runs both backends and either bypasses GitVersion's cache or uses a +cache populated during warm-up. Bypass is not a cold operating-system disk cache. +Each invocation is a fresh process; repeated invocations are not in-process JIT +warm-up. Case order rotates between repetitions to reduce ordering bias. + +GNU time records elapsed time, user/system CPU and peak RSS for each invocation. +The output includes all 15 samples per case, min/median/p95/max summaries, and +full version JSON. Every output must equal the same fixture's reference across +backends, runtimes and cache modes. Timing thresholds are intentionally absent; +exit failures, invalid metrics, runtime-selection failure and output differences +fail the experiment. Raw samples remain available after partial failures. + +Local Linux reproduction (Node.js, Git, GNU time, SDK 11 and runtimes 10/11 needed): + +```sh +dotnet publish src/GitVersion.App -c Release -f net10.0 --no-self-contained -o artifacts/runtime-app/net10 +dotnet publish src/GitVersion.App -c Release -f net11.0 --no-self-contained -o artifacts/runtime-app/net11 +node build/runtime-comparison.mjs artifacts/runtime-app/net10/gitversion.dll artifacts/runtime-app/net11/gitversion.dll artifacts/runtime-comparison +``` + +Use a new output directory on each run. For a quick harness smoke check set +`COMPARISON_REPETITIONS=2 COMPARISON_SIZES=10,30`; do not present smoke timings +as performance evidence. Download the workflow artifact to inspect `summary.json`, +`samples.json`, per-invocation output/metrics and host traces. + +This initial experiment measures direct framework-dependent CLI execution only, +including process startup. It does not isolate SDK tool dispatch, allocations, +GC, individual calculation phases or decompression. It does not cover cold disks, +Windows/macOS, alternate CPUs, shallow repositories or complex merge histories. +Shared hosted-runner noise and RC2 maturity limit conclusions. Repeat promising +results on a controlled runner and GA before proposing production optimizations; +profile object inflation before considering a span-decoder rewrite. diff --git a/build/runtime-comparison.mjs b/build/runtime-comparison.mjs new file mode 100644 index 0000000000..f8feb94264 --- /dev/null +++ b/build/runtime-comparison.mjs @@ -0,0 +1,148 @@ +#!/usr/bin/env node +// Linux-only, dependency-free fresh-process measurements. See runtime-comparison.md. +import { spawnSync } from 'node:child_process'; +import { mkdirSync, writeFileSync, readFileSync, existsSync } from 'node:fs'; +import { resolve, join } from 'node:path'; +import { cpus, release, arch } from 'node:os'; + +const [net10, net11, runtime10, runtime11, destination] = process.argv.slice(2); +if (!destination || !runtime10?.startsWith('10.') || !runtime11?.startsWith('11.')) { + throw new Error('Usage: node build/runtime-comparison.mjs '); +} +if (process.platform !== 'linux') throw new Error('Requires Linux and GNU /usr/bin/time.'); +const repetitions = Number(process.env.COMPARISON_REPETITIONS ?? 15); +const sizes = (process.env.COMPARISON_SIZES ?? '100,5000').split(',').map(Number); +if (!Number.isInteger(repetitions) || repetitions < 2 || repetitions > 100 || + sizes.length !== 2 || sizes.some(n => !Number.isInteger(n) || n < 2 || n > 50000)) { + throw new Error('Use 2–100 repetitions and two fixture sizes of 2–50000 commits.'); +} +const output = resolve(destination); +if (existsSync(output)) throw new Error(`Refusing to reuse output directory: ${output}`); +mkdirSync(output, { recursive: true }); +// Keep only process-location essentials: CI branch/agent variables and GitVersion +// overrides must not make fixture calculations depend on the hosting job. +const inherited = Object.fromEntries(Object.entries(process.env).filter(([key]) => + /^(PATH|HOME|TMPDIR|TMP|TEMP|DOTNET_ROOT(?:_[A-Z0-9]+)?)$/.test(key))); +const env = { ...inherited, LC_ALL: 'C', TZ: 'UTC', DOTNET_ROLL_FORWARD: 'Disable', + DOTNET_CLI_TELEMETRY_OPTOUT: '1', GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', GIT_TERMINAL_PROMPT: '0' }; +function run(command, args, options = {}) { + const result = spawnSync(command, args, { env, encoding: 'utf8', timeout: 120000, + maxBuffer: 32 * 1024 * 1024, ...options }); + if (result.error || result.status !== 0) { + throw new Error(`${command} failed (${result.status}): ${result.error ?? result.stderr}`); + } + return result.stdout.trim(); +} +function json(path, value) { writeFileSync(path, JSON.stringify(value, null, 2) + '\n'); } +const variants = [ + { name: 'net10-runtime10', dll: resolve(net10), runtime: runtime10 }, + { name: 'net10-runtime11', dll: resolve(net10), runtime: runtime11 }, + { name: 'net11-runtime11', dll: resolve(net11), runtime: runtime11 }, +]; +const metadata = { source: run('git', ['rev-parse', 'HEAD']), dirty: run('git', ['status', '--porcelain']), + sdk: run('dotnet', ['--version']), runtimes: run('dotnet', ['--list-runtimes']), + git: run('git', ['--version']), node: process.version, os: release(), arch: arch(), + cpu: cpus()[0]?.model, repetitions, sizes, variants }; +json(join(output, 'environment.json'), metadata); +for (const variant of variants) { + const trace = join(output, `${variant.name}.host.txt`); + run('dotnet', ['exec', '--fx-version', variant.runtime, variant.dll, '--version'], { + env: { ...env, COREHOST_TRACE: '1', COREHOST_TRACEFILE: trace, + DOTNET_HOST_TRACE: '1', DOTNET_HOST_TRACEFILE: trace }, + }); + const text = readFileSync(trace, 'utf8'); + const selected = text.split('\n').filter(line => line.includes('CoreCLR path =')); + if (!selected.some(line => line.includes(`/Microsoft.NETCore.App/${variant.runtime}/libcoreclr.so`))) { + throw new Error(`Could not verify loaded runtime ${variant.runtime}; inspect ${trace}`); + } +} + +const fixtures = []; +for (const [index, size] of sizes.entries()) { + const name = index === 0 ? 'small' : 'large'; + const path = join(output, name); + run('git', ['init', '--initial-branch=main', path]); + // Fixed identity, timestamps, content and parent chain yield reproducible object IDs. + let input = ''; + for (let i = 1; i <= size; i++) { + const content = `fixture ${i}\n`; + const message = `Commit ${i}\n`; + input += `commit refs/heads/main\nmark :${i}\ncommitter Fixture ${1700000000 + i} +0000\ndata ${Buffer.byteLength(message)}\n${message}`; + if (i > 1) input += `from :${i - 1}\n`; + input += `M 100644 inline fixture.txt\ndata ${Buffer.byteLength(content)}\n${content}\n`; + } + run('git', ['-C', path, 'fast-import', '--quiet'], { input }); + run('git', ['-C', path, 'reset', '--hard']); + run('git', ['-C', path, 'tag', '1.0.0', `main~${size - 1}`]); + run('git', ['-C', path, 'gc', '--aggressive', '--prune=now']); + fixtures.push({ name, path, size, head: run('git', ['-C', path, 'rev-parse', 'HEAD']) }); +} +json(join(output, 'fixtures.json'), fixtures); +const rows = []; +const references = new Map(); +function capture(fixture, backend, variant, cache, sample) { + const id = `${fixture.name}-${backend}-${variant.name}-${cache}-${sample}`; + const metrics = join(output, `${id}.time`); + const args = ['exec', '--fx-version', variant.runtime, variant.dll, fixture.path, + '--output', 'json', '--no-normalize', '--no-fetch', '--verbosity', 'Quiet']; + if (cache === 'bypass') args.push('--no-cache'); + const stdout = run('/usr/bin/time', ['-f', '%e,%U,%S,%M', '-o', metrics, 'dotnet', ...args], { + env: { ...env, GITVERSION_GIT_BACKEND: backend }, + }); + const parsed = JSON.parse(stdout); + const canonical = JSON.stringify(parsed, Object.keys(parsed).sort()); + if (references.has(fixture.name) && references.get(fixture.name) !== canonical) { + throw new Error(`Version output mismatch: ${id}`); + } + references.set(fixture.name, canonical); + json(join(output, `${id}.json`), parsed); + const [elapsedSeconds, userSeconds, systemSeconds, peakRssKiB] = readFileSync(metrics, 'utf8').trim().split(',').map(Number); + if (![elapsedSeconds, userSeconds, systemSeconds, peakRssKiB].every(Number.isFinite) || peakRssKiB <= 0) { + throw new Error(`Invalid GNU time metrics: ${metrics}`); + } + return { fixture: fixture.name, backend, variant: variant.name, cache, sample, + elapsedSeconds, userSeconds, systemSeconds, cpuSeconds: userSeconds + systemSeconds, peakRssKiB }; +} +for (const fixture of fixtures) { + const cases = ['libgit2', 'managed'].flatMap(backend => variants.flatMap(variant => + ['bypass', 'hit'].map(cache => ({ backend, variant, cache })))); + // Unmeasured warm-up includes cache population. Each sample is still a new process. + for (const c of cases) capture(fixture, c.backend, c.variant, c.cache, 'warmup'); + for (let sample = 0; sample < repetitions; sample++) { + // Rotate case order to distribute runner drift across variants. + for (let j = 0; j < cases.length; j++) { + const c = cases[(j + sample) % cases.length]; + rows.push(capture(fixture, c.backend, c.variant, c.cache, sample)); + json(join(output, 'samples.json'), rows); + } + } +} +const summary = []; +for (const row of rows) { + const key = [row.fixture, row.backend, row.variant, row.cache].join('/'); + if (summary.some(item => item.case === key)) continue; + const group = rows.filter(r => [r.fixture, r.backend, r.variant, r.cache].join('/') === key); + const stats = field => { + const values = group.map(r => r[field]).sort((a, b) => a - b); + const middle = Math.floor(values.length / 2); + return { min: values[0], median: values.length % 2 ? values[middle] : (values[middle - 1] + values[middle]) / 2, + p95: values[Math.ceil(values.length * .95) - 1], max: values.at(-1) }; + }; + summary.push({ case: key, samples: group.length, elapsedSeconds: stats('elapsedSeconds'), + userSeconds: stats('userSeconds'), systemSeconds: stats('systemSeconds'), cpuSeconds: stats('cpuSeconds'), peakRssKiB: stats('peakRssKiB') }); +} +json(join(output, 'summary.json'), summary); +writeFileSync(join(output, 'summary.md'), [ + '# Runtime comparison', + '', + 'Informational fresh-process measurements; shared-runner timings are not a performance gate.', + '', + '| Fixture / backend / build-runtime / cache | Samples | Median seconds | p95 seconds | Median CPU seconds | Median peak RSS KiB |', + '|---|---:|---:|---:|---:|---:|', + ...summary.map(s => `| ${s.case} | ${s.samples} | ${s.elapsedSeconds.median} | ${s.elapsedSeconds.p95} | ${s.cpuSeconds.median.toFixed(3)} | ${s.peakRssKiB.median} |`), + '', + 'All version JSON outputs matched. See the artifact for raw samples, runtime traces and environment metadata.', + '', +].join('\n')); +console.log(`Output parity verified across ${rows.length} samples. Results: ${output}`); From a4ac907dfe60e6c9ced8b229f99f885351cf8d47 Mon Sep 17 00:00:00 2001 From: Artur Stolear Date: Tue, 20 Oct 2026 01:07:41 +0200 Subject: [PATCH 2/2] build: retain mismatching runtime comparison output --- build/runtime-comparison.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/runtime-comparison.mjs b/build/runtime-comparison.mjs index f8feb94264..5ec2565e53 100644 --- a/build/runtime-comparison.mjs +++ b/build/runtime-comparison.mjs @@ -91,12 +91,12 @@ function capture(fixture, backend, variant, cache, sample) { env: { ...env, GITVERSION_GIT_BACKEND: backend }, }); const parsed = JSON.parse(stdout); + json(join(output, `${id}.json`), parsed); const canonical = JSON.stringify(parsed, Object.keys(parsed).sort()); if (references.has(fixture.name) && references.get(fixture.name) !== canonical) { throw new Error(`Version output mismatch: ${id}`); } references.set(fixture.name, canonical); - json(join(output, `${id}.json`), parsed); const [elapsedSeconds, userSeconds, systemSeconds, peakRssKiB] = readFileSync(metrics, 'utf8').trim().split(',').map(Number); if (![elapsedSeconds, userSeconds, systemSeconds, peakRssKiB].every(Number.isFinite) || peakRssKiB <= 0) { throw new Error(`Invalid GNU time metrics: ${metrics}`);