diff --git a/.github/workflows/REVIEW.md b/.github/workflows/REVIEW.md index 2209d47543..a385ce8870 100644 --- a/.github/workflows/REVIEW.md +++ b/.github/workflows/REVIEW.md @@ -3,21 +3,26 @@ **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `skills/internal/preflight.md` (repo root). -A per-PR gate step is a workflow step that runs on `pull_request` and fails the lane when it -finds a defect; each one enforces its rule automatically, with no reviewer involved -(`skills/internal/preflight.md` sec."doc.yml - the gates", sec."extended_checks.yml"). +**A diff that weakens a per-PR check - a step, a matrix cell, or a workflow's `pull_request` +trigger that runs on every pull request and fails the lane when it finds a defect - is a +defect: deleting it, stopping its failure from failing the lane (`continue-on-error`, a +trailing `|| true`, a swallowed exit code), shrinking what it checks, or narrowing its +condition to anything but a `matrix.role` condition that still runs it on every pull request +or the nightly cron.** -**A diff that weakens a per-PR gate step - deletes it, stops its failure failing the lane -(`continue-on-error`, a trailing `|| true`, a swallowed exit code, a narrowed `if:`), or -shrinks what it checks - is a defect; a step the diff adds or changes makes its failure the -lane's failure.** +**A per-PR check the diff adds fails the lane when it finds a defect.** -**A diff that adds or changes a per-PR gate step states a run of the command the diff adds or -changes, on the lane's platform, in its PR body or commit message; a green run of that lane on -the PR's head commit is that evidence.** A step that fails for a non-defect turns a green -branch red for everyone. +**A per-PR check leaves the per-PR path only to the nightly cron (`github.event_name == +'schedule' || github.event_name == 'workflow_dispatch'`), and the diff either names the +preflight gate - a check `preflight` runs locally before a push - that keeps it per PR +(`skills/internal/preflight.md` sec."extended_checks.yml") or states the platform no per-PR +cell has.** A per-PR job fits 35 minutes; what does not fit moves. -**A diff that adds or changes a step in `pages.yml` that names the deployed games writes -that list as a `for g in ; do` loop.** `examples/games/REVIEW.das` (repo root) reads -the deployed game list from those loops, so a list spelled any other way is one nothing -cross-checks. +**A diff that adds or changes a per-PR check states a run of that check's command, on the +lane's platform, in its PR body or commit message; a green run of that lane on the PR's head +commit is that evidence.** A check that fails for a non-defect turns a green branch red for +everyone. + +**A step in `pages.yml` that names the deployed games spells the list as a `for g in ; do` +loop, never inline.** `examples/games/REVIEW.das` (repo root) reads the deployed list from +those loops; an inline list beside a surviving loop is one nothing cross-checks. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8d9f81e450..04402fa220 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,14 +9,16 @@ on: # Nightly: runs everything the per-PR lanes deliberately skip — # * the heavy Windows toolchain builds (mingw, clang-cl), gated OFF per-PR # below (see their `if:`), including their FULL AOT sweeps; - # * the main matrix (Release cells carry the FULL test_aot build + AOT - # suite — "Slow Release Tests"; Debug cells ride along, matrix can't be - # filtered in a job-level `if`). Per-PR CI only builds test_aot_subset - # (tests/language, part of ALL) as a compile+link gate and runs no AOT - # tests at all. + # * the main matrix in its nightly shape (ci/ci_matrix.py): the per-PR + # cells plus the sanitizer cells and windows 64 Debug, each a 40-55 + # minute job over the 35-minute PR budget; the Release cells carry the + # FULL test_aot build + AOT suite — "Slow Release Tests". Per-PR CI only + # builds test_aot_subset (tests/language, part of ALL) as a compile+link + # gate and runs no AOT tests at all (preflight --full's tests-aot lane is + # the pre-push AOT sweep). # bundle_smoke / gcc stay per-PR-only. A manual `workflow_dispatch` runs - # the FULL workflow: every per-PR job, both toolchains, and the full AOT - # suite. + # the FULL workflow: every per-PR job, both toolchains, the nightly cells, + # and the full AOT suite. # 02:00 UTC: dead time for EU working hours — the nightly is ~12 jobs and # must not compete with daytime PR lanes for runners. - cron: '0 2 * * *' @@ -38,6 +40,7 @@ jobs: # Map a step output to a job output outputs: should_skip: ${{ steps.skip_check.outputs.should_skip }} + matrix: ${{ steps.matrix.outputs.matrix }} steps: - id: skip_check uses: fkirc/skip-duplicate-actions@v5 @@ -46,6 +49,17 @@ jobs: concurrent_skipping: 'same_content' do_not_skip: '["pull_request", "workflow_dispatch", "release"]' + # The build matrix is data in ci/ci_matrix.py, evaluated per event here (ci/test_ci_matrix.py pins it). + - uses: actions/checkout@v4 + with: + sparse-checkout: ci + - id: matrix + # two statements: a substitution inside echo would hide the script's exit code + run: | + set -eu + matrix=$(python3 ci/ci_matrix.py build '${{ github.event_name }}') + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + - name: Cache LLVM uses: actions/cache@v3 id: llvm-cache @@ -62,13 +76,12 @@ jobs: # `schedule` cron the matrix runs too — the Release cells carry the full # AOT suite (see "Slow Release Tests") — unconditionally on the canonical # repo (no pre_job skip: master unchanged overnight must still produce the - # nightly AOT signal). Debug cells ride along at night without an AOT step; - # `matrix` context is not available in a job-level `if`, so they can't be - # excluded here (actions runs with a matrix clause fail as a workflow-file - # error before any job spawns). + # nightly AOT signal). The cell list is per event - ci/ci_matrix.py, read + # through pre_job - because `matrix` is not available in a job-level `if`. if: >- (github.event_name == 'schedule' && github.repository == 'GaijinEntertainment/daScript') || (github.event_name != 'schedule' && needs.pre_job.outputs.should_skip != 'true') + name: "build (${{ matrix.target }}, ${{ matrix.architecture }}, ${{ matrix.cmake_preset }}, ${{ matrix.sanitizers }})" runs-on: ${{ matrix.runner }} # actions: write needed for `gh cache delete` in the sccache refresh step. permissions: @@ -79,159 +92,14 @@ jobs: das_llvm_disabled: 'OFF' strategy: fail-fast: false - matrix: - target: [linux, linux_arm, darwin15, darwin26, windows] - architecture: [32, 64, arm64] - cmake_preset: [ Debug, Release ] - sanitizers: [none] - - include: - - target: linux - release_target: linux - release_arch: x86_64 - runner: ubuntu-latest - archive_ext: tar.gz - sanitizers: none - - # Sanitizer matrix — Linux + Release + full tests (dastest + test_aot - # + ctest). Release keeps walltime in budget; the previous Debug+ASAN - # was prohibitive for production CI. See issue #2530. - - target: linux - architecture: 64 - runner: ubuntu-latest - cmake_preset: Release - sanitizers: asan - build_name: linux_asan - build_system: cmake - cmake_generator: Ninja - - - target: linux - architecture: 64 - runner: ubuntu-latest - cmake_preset: Release - sanitizers: tsan - build_name: linux_tsan - build_system: cmake - cmake_generator: Ninja - - - target: linux - architecture: 64 - runner: ubuntu-latest - cmake_preset: Release - sanitizers: ubsan - build_name: linux_ubsan - build_system: cmake - cmake_generator: Ninja - - - target: linux_arm - release_target: linux - release_arch: arm64 - runner: ubuntu-24.04-arm - archive_ext: tar.gz - - - target: darwin15 - architecture: arm64 - release_target: darwin15 - release_arch: arm64 - runner: macos-15 # Apple Silicon arm64, M2 - architecture_string: arm64 - archive_ext: tar.gz - - - target: darwin26 - release_target: darwin26 - release_arch: arm64 - runner: macos-26 # Apple Silicon arm64, M2 - architecture_string: arm64 - archive_ext: tar.gz - - - target: windows - runner: windows-latest - archive_ext: zip - - - target: windows - build_system: cmake - cmake_generator: Ninja - - - target: darwin15 - build_system: cmake - cmake_generator: Ninja - - - target: darwin26 - build_system: cmake - cmake_generator: Ninja - - - target: linux - build_system: cmake - cmake_generator: Ninja - - - target: linux_arm - build_system: cmake - cmake_generator: Ninja - - - target: windows - release_target: windows - release_arch: x86 - architecture: 32 - architecture_string: Win32 - - - target: windows - release_target: windows - release_arch: x86_64 - architecture: 64 - architecture_string: x64 - - # The regular Win64 Release lane is the fast MSVC compile/interpreter - # gate. LLVM + the JIT sweep are preserved by the nightly copy below. - - target: windows - architecture: 64 - cmake_preset: Release - llvm_disabled: 'ON' - jit_disabled: 'ON' - - # Debug Win64 also skips JIT. RelWithDebInfo is a separate - # nightly-only memory/leak diagnostic. - - target: windows - architecture: 64 - cmake_preset: Debug - jit_disabled: 'ON' - - exclude: - # macOS Intel (darwin15 x86_64) has no prebuilt LLVM.dll (the - # llvm-release workflow doesn't build one) and Apple no longer makes - # new Intel macOS runners worth gating on. Drop entirely. - - target: darwin15 - architecture: 64 - - - target: darwin15 - architecture: 32 - - - target: darwin26 - architecture: 32 - - - target: darwin26 - architecture: 64 - - - target: linux - architecture: 32 - - - target: linux - architecture: arm64 - - - target: linux_arm - architecture: 32 - - - target: linux_arm - architecture: arm64 # todo - - - target: windows - architecture: arm64 # todo https://github.com/actions/partner-runner-images/tree/main?tab=readme-ov-file#available-images - - # win32 Debug outgrew its test budgets (the interpreter sweep double-times-out at - # 1800s + 5400s on master's own scheduled runs, ~2.5h wall before failing) and the - # 32-bit rail is legacy — win32 Release stays as the 32-bit compile+test gate. - - target: windows - architecture: 32 - cmake_preset: Debug + # ci/ci_matrix.py build : Release + Debug on linux, linux_arm, darwin15, + # darwin26; windows 32 Release (the 32-bit compile+test gate - win32 Debug outgrew + # its test budgets) and windows 64 Release (the fast MSVC gate, LLVM and JIT off; + # build_windows_release_llvm_nightly keeps the JIT sweep). The nightly cron and a + # manual dispatch add the sanitizer cells (linux Release asan/tsan/ubsan, full + # tests) and windows 64 Debug, tagged nightly_only. No darwin x86_64: no prebuilt + # LLVM.dll for it, and no Intel runner worth gating on. + matrix: ${{ fromJSON(needs.pre_job.outputs.matrix) }} steps: - name: "SCM Checkout" @@ -440,14 +308,16 @@ jobs: ;; esac + # A nightly-only cell saves no slot: nothing restores it (the cron does not + # restore, and no PR lane runs the cell), so the tarball would only hold quota. - name: "Refresh sccache slot" - if: github.ref == 'refs/heads/master' + if: github.ref == 'refs/heads/master' && matrix.nightly_only != 'ON' env: GH_TOKEN: ${{ github.token }} SCKEY: sccache-${{ matrix.target }}-${{ matrix.architecture }}-${{ matrix.cmake_preset }}-${{ matrix.sanitizers }} run: gh cache delete "$SCKEY" -R ${{ github.repository }} || true - name: "Save sccache objects" - if: github.ref == 'refs/heads/master' + if: github.ref == 'refs/heads/master' && matrix.nightly_only != 'ON' uses: actions/cache/save@v4 with: path: ${{ runner.temp }}/sccache diff --git a/.github/workflows/extended_checks.yml b/.github/workflows/extended_checks.yml index c1cd0bee88..0c79b171ee 100644 --- a/.github/workflows/extended_checks.yml +++ b/.github/workflows/extended_checks.yml @@ -34,12 +34,23 @@ jobs: runs-on: ubuntu-latest outputs: should_skip: ${{ steps.skip_check.outputs.should_skip }} + matrix: ${{ steps.matrix.outputs.matrix }} steps: - id: skip_check uses: fkirc/skip-duplicate-actions@v5 with: concurrent_skipping: 'same_content' do_not_skip: '["pull_request", "workflow_dispatch"]' + # The job matrix is data in ci/ci_matrix.py, evaluated per event here (ci/test_ci_matrix.py pins it). + - uses: actions/checkout@v4 + with: + sparse-checkout: ci + - id: matrix + # two statements: a substitution inside echo would hide the script's exit code + run: | + set -eu + matrix=$(python3 ci/ci_matrix.py extended '${{ github.event_name }}') + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" ########################################################### extended_checks: @@ -50,6 +61,7 @@ jobs: if: >- (github.event_name == 'schedule' && github.repository == 'GaijinEntertainment/daScript') || (github.event_name != 'schedule' && needs.pre_job.outputs.should_skip != 'true') + name: "extended_checks (${{ matrix.target }}, ${{ matrix.role }})" runs-on: ${{ matrix.runner }} # actions: write needed for `gh cache delete` in the sccache refresh step. permissions: @@ -57,26 +69,18 @@ jobs: actions: write strategy: fail-fast: false - # The windows lane runs on the nightly cron and manual dispatch only — - # retired from the per-PR path: it re-runs the same shared das surface - # MSVC-slower while adding no windows-only checks (build.yml's windows - # cells are the per-PR MSVC gate), and it runs uncached by design (see - # the sccache comment below). The event ternary appears on BOTH the - # target axis and the include list: an include whose target is absent - # from the axis would be ADDED as a standalone cell, not dropped. - matrix: - target: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && fromJSON('["linux", "darwin15", "windows"]') || fromJSON('["linux", "darwin15"]') }} - architecture: [64, arm64] - - include: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && fromJSON('[{"target":"linux","runner":"ubuntu-latest","build_system":"cmake","cmake_generator":"Ninja"},{"target":"darwin15","architecture":"arm64","runner":"macos-15","architecture_string":"arm64","build_system":"cmake","cmake_generator":"Ninja"},{"target":"windows","runner":"windows-latest","build_system":"cmake","cmake_generator":"Ninja","architecture_string":"x64"}]') || fromJSON('[{"target":"linux","runner":"ubuntu-latest","build_system":"cmake","cmake_generator":"Ninja"},{"target":"darwin15","architecture":"arm64","runner":"macos-15","architecture_string":"arm64","build_system":"cmake","cmake_generator":"Ninja"}]') }} - - exclude: - - target: linux - architecture: arm64 - - target: windows - architecture: arm64 - - target: darwin15 - architecture: 64 + # The cells come from ci/ci_matrix.py (pre_job evaluates it for this event). Per PR: two + # darwin15 jobs split by role, each inside the 35-minute budget - `core` runs the tree's + # own gates (formatter, lint, ast-verify, dasgen, ci-das, REVIEW.das, the python gates, + # utils tests, the standalone exes), `modules` the module and service suites (ser/deser, + # MCP, dasllama-server, facade lint, sequence, dasweb, ladder, boulder-dash). A step's role + # condition is always `matrix.role != ''`, so the nightly `all` role runs every + # step - one job each on linux, darwin15 and windows, plus the steps too slow for a PR + # (tutorial dry-runs, the run form of examples, daslang_static, coverage, nano). The + # windows lane is nightly-only: it re-runs the same shared das surface MSVC-slower while + # adding no windows-only checks (build.yml's windows cells are the per-PR MSVC gate), and + # it runs uncached by design (see the sccache comment below). + matrix: ${{ fromJSON(needs.pre_job.outputs.matrix) }} steps: - name: "SCM Checkout" @@ -188,10 +192,11 @@ jobs: # The launcher exported via env also wraps the GLFW/libhv # ExternalProject sub-builds — harmless on gcc/clang. # SCCACHE_CACHE_SIZE bounds the slot: the default is 10G, so an unbounded - # slot grows until it evicts other caches in the repository. 500M holds a - # full linux/macOS object set with room — build.yml's linux Release slot - # sits at ~94MB. - echo "SCCACHE_CACHE_SIZE=500M" >> $GITHUB_ENV + # slot grows until it evicts other caches in the repository. A Release + # build of daslang + daslang_static + the sequence smoke's module targets + # is ~800MB of objects; 1500M holds all of it, so the nightly save lands + # a complete set and a PR lane's build is a cache read. + echo "SCCACHE_CACHE_SIZE=1500M" >> $GITHUB_ENV echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> $GITHUB_ENV echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> $GITHUB_ENV # Stable per-config key in its own `sccache-extended-*` namespace so it never @@ -253,13 +258,13 @@ jobs: esac - name: "Refresh sccache slot" - if: matrix.target != 'windows' && github.ref == 'refs/heads/master' + if: matrix.target != 'windows' && matrix.role != 'modules' && github.ref == 'refs/heads/master' env: GH_TOKEN: ${{ github.token }} SCKEY: sccache-extended-${{ matrix.target }}-${{ matrix.architecture }} run: gh cache delete "$SCKEY" -R ${{ github.repository }} || true - name: "Save sccache objects" - if: matrix.target != 'windows' && github.ref == 'refs/heads/master' + if: matrix.target != 'windows' && matrix.role != 'modules' && github.ref == 'refs/heads/master' uses: actions/cache/save@v4 with: path: ${{ runner.temp }}/sccache @@ -271,15 +276,23 @@ jobs: run: npm install -g @ast-grep/cli tree-sitter-cli@0.26.8 - name: "Run dasgen and check generated files are up to date" + if: matrix.role != 'modules' run: cmake --build ./build --config Release --target check_dasgen - name: "Run examples from modules" + # Nightly: the run form costs 5-8 minutes; per PR, preflight's compile-sweep gate + # compiles every example and tutorial root, and the ci-das step below covers the + # CI-only surface. + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: cmake --build ./build --config Release --target run_examples - name: "Run utils tests" + if: matrix.role != 'modules' run: cmake --build ./build --config Release --target run_utils_tests - name: "Run tutorial dry-runs" + # Nightly, for the same reason as the examples' run form (8-11 minutes per lane). + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: cmake --build ./build --config Release --target dry_run_tutorials - name: "Check the example games stay wired to the site" @@ -288,8 +301,8 @@ jobs: # step, the examples-page card and its poster, the playground sample and the # interpreted fallback's file bundle. All of that is hand-maintained, and the # deploy that consumes it only runs on master, so a mismatch is invisible until - # the site is already broken. Linux only: it reads files, nothing platform-bound. - if: matrix.target == 'linux' + # the site is already broken. The core role: it reads files, nothing platform-bound. + if: matrix.role != 'modules' run: $BIN/daslang examples/games/REVIEW.das - name: "Verify authored-doc code blocks (nightly only)" @@ -301,9 +314,10 @@ jobs: run: $BIN/daslang ./utils/internal/doc-verify/main.das -- --daslang $BIN/daslang - name: "Cross-compile nano for cortex-m4" - # nano's drift tripwire - the only freestanding build in CI. Linux only: - # one arch catches the drift, and the toolchain is an apt away. - if: matrix.target == 'linux' + # nano's drift tripwire - the only freestanding build in CI. Nightly, on the linux + # cell: the toolchain is an apt package no darwin cell (the per-PR platform) and no + # developer box carries, so no per-PR cell and no preflight gate can run it. + if: matrix.target == 'linux' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') run: | set -eux # newlib and its libstdc++ are only *recommended* by gcc-arm-none-eabi, @@ -313,6 +327,8 @@ jobs: ci/nano_arm_build.sh "$BIN/daslang" - name: "Build standalone executables" + # core: the formatter and lint steps below run the das-fmt / das-lint exes this builds + if: matrix.role != 'modules' run: | set -eux cmake --build ./build --config Release --target all_utils_exe @@ -320,6 +336,7 @@ jobs: $BIN/daslang -exe -output ./bin/das-lint ./utils/lint/main.das - name: "Sequence release smoke test" + if: matrix.role != 'core' # Full daspkg install -> release -> launch cycle on the in-tree sequence # game. Exercises release_include_dll + the loader fixes (Windows # LoadLibraryEx, POSIX rpath $ORIGIN / @loader_path) end-to-end. @@ -359,6 +376,7 @@ jobs: esac - name: "Run formatter" + if: matrix.role != 'modules' run: | set -eux cmake --build ./build --config Release --target check_format @@ -366,14 +384,16 @@ jobs: $BIN/das-fmt.exe --path ./ --verify --exclude-mask build/ - name: "Run lint on changed .das and .md files" - if: matrix.target == 'linux' + if: matrix.role != 'modules' run: | set -eux BASE_REF="${{ github.event.pull_request.base.ref }}" BASE_REF="${BASE_REF:-master}" # a changed .md arms its folder's document rules (LINT025/026/027) with no .das compiled; # a repo-root .md arms the whole tree - intended, and a few seconds with nothing to compile - mapfile -t CHANGED < <(git diff --name-only --diff-filter=AM "origin/${BASE_REF}...HEAD" -- '*.das' '*.md') + # (a read loop, not mapfile: the darwin runner's /bin/bash is 3.2) + CHANGED=() + while IFS= read -r f; do CHANGED+=("$f"); done < <(git diff --name-only --diff-filter=AM "origin/${BASE_REF}...HEAD" -- '*.das' '*.md') if [ ${#CHANGED[@]} -eq 0 ]; then echo "no .das or .md files changed; skipping lint" exit 0 @@ -386,7 +406,7 @@ jobs: $BIN/das-lint.exe "${CHANGED[@]}" --quiet --disable LINT019 - name: "Run ast-verify on changed .das files" - if: matrix.target == 'linux' + if: matrix.role != 'modules' # a bare exit 143 with no finding is the runner killing the step (memory: each item is a # whole-engine compile under the verifier) - the explicit deadline makes that a named failure timeout-minutes: 90 @@ -394,7 +414,8 @@ jobs: set -eux BASE_REF="${{ github.event.pull_request.base.ref }}" BASE_REF="${BASE_REF:-master}" - mapfile -t CHANGED < <(git diff --name-only --diff-filter=AM "origin/${BASE_REF}...HEAD" -- '*.das') + CHANGED=() + while IFS= read -r f; do CHANGED+=("$f"); done < <(git diff --name-only --diff-filter=AM "origin/${BASE_REF}...HEAD" -- '*.das') # canary: macro-PRODUCED AST is only verified where a macro expands, and the tree-wide # sweep is nightly-only - so the heaviest qmacro consumer verifies on every PR CHANGED+=("tests/linq/test_linq_fold.das") @@ -409,7 +430,9 @@ jobs: f="$1" case "$(basename "$f")" in cant_*|failed_*|invalid_*) exit 0 ;; esac case "$f" in *ast-fuzz/selftest/*) exit 0 ;; esac # deliberately broken fixtures; test_ast_fuzz.das owns them - out=$(timeout 300 "$BIN/daslang" -dry-run --ast-verify-batch "$f" 2>&1); rc=$? + # perl's alarm survives the exec, so the deadline holds on darwin too (no coreutils timeout there); + # a compile the alarm kills exits 142 (SIGALRM), GNU timeout's 124 kept for the linux nightly + out=$(perl -e 'alarm shift; exec @ARGV' 300 "$BIN/daslang" -dry-run --ast-verify-batch "$f" 2>&1); rc=$? if grep -q 'error\[20510\]' <<< "$out"; then # the verifier's own require closure (daslib/ast, ast_boost, ...) cannot be a main file # under the force-include - say so instead of counting a non-run as clean @@ -421,7 +444,7 @@ jobs: grep 'AST verify' <<< "$out" | sort -u | head -5 exit 1 fi - if [ "$rc" -eq 124 ]; then + if [ "$rc" -eq 124 ] || [ "$rc" -eq 142 ]; then echo "ast-verify timed out (300s) on $f" exit 1 fi @@ -442,10 +465,13 @@ jobs: printf '%s\0' "${CHANGED[@]}" | xargs -0 -P 2 -n 1 /tmp/ast_verify_changed.sh - name: "Test daslang_static" + # modules role, per PR: static-registration / no-dynamic-modules divergence has no local mirror + if: matrix.role != 'core' run: | set -eux $BIN/daslang_static _dasroot_/dastest/dastest.das -- --color --failures-only --test ./tests - name: "Test ser/deser" + if: matrix.role != 'core' run: | $BIN/daslang _dasroot_/dastest/dastest.das -- --color --failures-only --test ./tests --ser serialized.bin $BIN/daslang _dasroot_/dastest/dastest.das -- --color --failures-only --test ./tests --deser serialized.bin @@ -495,46 +521,57 @@ jobs: | xargs -0 -P "$(nproc)" -n 1 /tmp/ast_verify_one.sh - name: "Test MCP tools" - if: matrix.target == 'linux' + if: matrix.role != 'core' run: | set -eux + # the ast-grep tools need the daslang grammar library and the sgconfig.yml its post-build + # step stamps - run_utils_tests builds it in the core role, this job must build it itself + cmake --build ./build --config Release --target tree_sitter_daslang $BIN/daslang _dasroot_/dastest/dastest.das -- --color --failures-only --test ./utils/mcp/test_tools.das $BIN/daslang _dasroot_/dastest/dastest.das -- --color --failures-only --test ./utils/mcp/test_crosstree_guard.das - name: "Test LSP cross-tree guard (python)" - if: matrix.target == 'linux' + if: matrix.role != 'modules' run: | set -eux python3 utils/lsp/test_crosstree_guard.py + - name: "Test CI matrix and role conditions (python)" + # ci/ci_matrix.py is what pre_job fed this very run; the test pins its per-event cells and + # the `matrix.role != ''` spelling of every step condition in this workflow + if: matrix.role != 'modules' + run: | + set -eux + python3 ci/test_ci_matrix.py + - name: "Test shipped-skills gate (python)" - if: matrix.target == 'linux' + if: matrix.role != 'modules' run: | set -eux python3 ci/test_check_shipped_skills.py - name: "Markdown ASCII gate (python)" - if: matrix.target == 'linux' + if: matrix.role != 'modules' run: | set -eux python3 ci/test_fix_md_ascii.py python3 ci/fix_md_ascii.py --check - name: "Test watchdog tray wording (python)" - if: matrix.target == 'linux' + if: matrix.role != 'modules' run: | set -eux python3 utils/watchdog/test_tray_state.py python3 utils/watchdog/test_consent.py - name: "Test pip wheel repack (python)" - if: matrix.target == 'linux' + if: matrix.role != 'modules' run: | set -eux python3 ci/test_wheel_build.py - name: "Test boulder-dash samples (both copies)" - if: matrix.target == 'linux' + if: matrix.role != 'core' run: | set -eux # dastest walks the folder: every test_*.das runs, files without [test] @@ -543,7 +580,7 @@ jobs: $BIN/daslang _dasroot_/dastest/dastest.das -- --color --failures-only --test ./web/examples/ui/samples/examples/boulder-dash - name: "Test review-md discovery" - if: matrix.target == 'linux' + if: matrix.role != 'modules' run: | set -eux $BIN/daslang _dasroot_/dastest/dastest.das -- --color --failures-only --test ./utils/internal/review-md/test_scan.das @@ -555,13 +592,13 @@ jobs: - name: "Test dastest own suite" # the whole directory, so a new dastest/tests file is covered with no # hand-added row (this also runs the review_gate library tests) - if: matrix.target == 'linux' + if: matrix.role != 'modules' run: | set -eux $BIN/daslang _dasroot_/dastest/dastest.das -- --color --failures-only --test ./dastest/tests - name: "Test benchmark results updater" - if: matrix.target == 'linux' + if: matrix.role != 'modules' run: | set -eux $BIN/daslang _dasroot_/dastest/dastest.das -- --color --failures-only --test ./benchmarks/sql/tests/test_update_results.das @@ -569,7 +606,7 @@ jobs: - name: "Run REVIEW.das gates" # the CI counterpart of the make_pr step-0a walk (fail-fix: agents only # ever review a mechanically-clean tree) - if: matrix.target == 'linux' + if: matrix.role != 'modules' run: | set -eux $BIN/daslang ./utils/internal/review-md/all.das @@ -578,17 +615,19 @@ jobs: # the deploy runs the same generator; this is the per-PR half - a committed # index.html/feed.xml/sitemap.xml that the generator would rewrite is a red lane, # and so is a page missing its title, description, OpenGraph tags or Atom link - if: matrix.target == 'linux' + if: matrix.role != 'modules' run: | set -eux - pip install markdown + # into the interpreter that runs the script: on the darwin runner `pip` and `python3` differ, + # and a Homebrew python refuses a system-site install without the last flag + python3 -m pip install markdown || python3 -m pip install --user markdown || python3 -m pip install --break-system-packages markdown python3 site-dasllama/build_news.py --root site-dasllama git diff --exit-code -- site-dasllama/index.html site-dasllama/stories.html site-dasllama/stories site-dasllama/feed.xml site-dasllama/sitemap.xml test -z "$(git status --porcelain -- site-dasllama/stories)" # a story page the generator wrote but nobody committed python3 site-dasllama/test_metadata.py - name: "Test pr-babysit verdict core" - if: matrix.target == 'linux' + if: matrix.role != 'modules' run: | set -eux $BIN/daslang _dasroot_/dastest/dastest.das -- --color --failures-only --test ./utils/internal/pr-babysit/test_watch_verdict.das @@ -599,7 +638,7 @@ jobs: # the dasHV + dasAudio + dasllama facade surface the server reaches, # catching any API/module breakage in CI. The exchange-client, catalog, # and setup-mode suites are model-free and run fully (loopback only). - if: matrix.target == 'linux' + if: matrix.role != 'core' run: | set -eux $BIN/daslang _dasroot_/dastest/dastest.das -- --compile-only --color --failures-only --test ./utils/dasllama-server/test_openai_server.das @@ -618,7 +657,7 @@ jobs: # ban, declaration coverage, generated-ENVIRONMENT.md drift. Model-free, # sub-second — this lane is what makes each REVIEW's "the registry test # enforces it" sentence true. - if: matrix.target == 'linux' + if: matrix.role != 'core' run: | set -eux $BIN/daslang _dasroot_/dastest/dastest.das -- --color --failures-only --test ./modules/dasLLAMA/tests/test_env_registry.das @@ -630,7 +669,7 @@ jobs: # aarch64, and the cells pin vecmath's answers - log2(0) is -127 there, while # the x64 JIT keeps @llvm.log2 and returns -inf. Needs -jit or the emitters # never run. Model-free, ~2 seconds. - if: matrix.target == 'darwin15' + if: matrix.target == 'darwin15' && matrix.role != 'core' run: | set -eux $BIN/daslang -jit _dasroot_/dastest/dastest.das -- --color --failures-only --test ./modules/dasLLVM/tests/llvm_vector_math.das @@ -638,7 +677,7 @@ jobs: - name: "Test dasllama facade lint" # The DASLLAMA001 smokes: a direct engine require kabooms (expected-compile-failure # file), and the per-file options escape admits one. Model-free, sub-second. - if: matrix.target == 'linux' + if: matrix.role != 'core' run: | set -eux $BIN/daslang -jit _dasroot_/dastest/dastest.das -- --color --failures-only --test ./modules/dasLLAMA/tests/failed_dasllama_lint_require.das @@ -652,7 +691,7 @@ jobs: # ./tests` never reaches them — without this step the store, importer, # config and HTTP tests run only by hand. The server suites bind # 127.0.0.1:19011/19012/19013 and shut themselves down. - if: matrix.target == 'linux' + if: matrix.role != 'core' run: | set -eux for suite in test_samples_store test_curated_import test_playground_config test_playground_server test_build_queue test_build_artifacts test_build_endpoints; do @@ -663,7 +702,7 @@ jobs: # The dasllama.io exchange service (sidecars + ladder records). In-dir # suites like the playground's; the server suite binds 127.0.0.1:19015 # and shuts itself down, the store/config suites run against :memory:. - if: matrix.target == 'linux' + if: matrix.role != 'core' run: | set -eux for suite in test_ladder_store test_ladder_config test_ladder_server; do @@ -674,7 +713,7 @@ jobs: # The wasm build worker (compute-box side of the same pipeline). In-dir # suites like the playground's; the client suite runs a stub playground # on 127.0.0.1:19014 and shuts itself down. - if: matrix.target == 'linux' + if: matrix.role != 'core' run: | set -eux for suite in test_buildd_config test_buildd_core test_buildd_client test_roll_toolchain; do @@ -688,7 +727,7 @@ jobs: # (a require breaking, a manifest entry pointing at a moved file) # before the site serves it. The browser leg against the live site is # a nightly (nightly_playground.yml), not a per-PR check. - if: matrix.target == 'linux' + if: matrix.role != 'core' run: | set -eux $BIN/daslang _dasroot_/dastest/dastest.das -- --color --failures-only --test ./utils/internal/dasweb-verify/test_verify_core.das @@ -710,12 +749,14 @@ jobs: # dasOpenGL's GLFW-gated files and the imgui-consuming tools # (jobque-timeline, the dasHerd UI root). Single source of truth with the # local mirror: utils/internal/preflight's ci-das gate reads the same list file. + if: matrix.role != 'modules' run: | set -eux $BIN/daslang utils/internal/preflight/main.das -- --only ci-das - name: "Coverage" - if: matrix.target == 'linux' + # a report, not a gate (dascov fails on nothing it measures); nightly on the linux cell + if: matrix.target == 'linux' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') run: | $BIN/daslang dastest/dastest.das -- --cov-path coverage.lcov --color --test ./tests/language --timeout 1800 $BIN/dascov.exe -- coverage.lcov --exclude tests/language diff --git a/ci/REVIEW.das b/ci/REVIEW.das new file mode 100644 index 0000000000..d3777235b9 --- /dev/null +++ b/ci/REVIEW.das @@ -0,0 +1,48 @@ +options gen2 + +require strings +require daslib/strings_boost +require daslib/fio +require dastest/review_gate + +// The mechanical half of ci/REVIEW.md (contract: REVIEW_COMMON.md at the repo root). +// Run from the repo root: bin/daslang ci/REVIEW.das - exit 0 clean, 1 with findings. + +def private read_text(path : string) : string { + var text = "" + fopen(path, "rb") $(f) { + if (f != null) { + text = fread(f) + } + } + return text +} + +def private workflows_text() : string { + var all : array + dir(".github/workflows") $(name) { + return if (!(name |> ends_with(".yml"))) + all |> push(read_text(path_join(".github/workflows", name))) + } + return join(all, "\n") +} + +//! every ci/test_*.py is named by a workflow step - a pin no row runs never fires +def private check_tests_wired(workflows : string) { + dir("ci") $(name) { + return if (!(name |> starts_with("test_")) || !(name |> ends_with(".py"))) + if (find(workflows, "ci/{name}") < 0) { + gate_finding("ci/{name}", "no .github/workflows step names ci/{name} - a test no CI row runs never runs again; add the step (extended_checks.yml's core role carries the python gates)") + } + } +} + +[export] +def main() : int { + if (!fexist("ci/REVIEW.das") || !stat(".github/workflows").is_dir) { + to_log(LOG_ERROR, "ci/REVIEW.das: run from the repo root\n") + return 2 + } + check_tests_wired(workflows_text()) + return gate_verdict("ci") +} diff --git a/ci/REVIEW.md b/ci/REVIEW.md index 964392c68c..45d8fb5808 100644 --- a/ci/REVIEW.md +++ b/ci/REVIEW.md @@ -1,8 +1,15 @@ # CI Scripts Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture -doc: `CLAUDE.md` (repo root). +docs: `CLAUDE.md` (repo root), `skills/internal/preflight.md`. **A diff that shrinks what the bundle gate rejects - `smoke_test_bundle.sh` and the checkers it runs (`check_shipped_skills.py`) - is a defect**: every bundle it failed before the diff still fails. A new `--exclude` or skip may name only a file no check flagged before the diff. + +**Weakening `ci/test_ci_matrix.py` - dropping or loosening any assertion it makes - is a +defect.** Those assertions are what turns a job or step that stopped running per PR into a red +test. + +**Weakening `REVIEW.das` (beside this file) is a defect: dropping a check, narrowing what a check +walks, or rewriting a finding text so it no longer names what failed.** diff --git a/ci/ci_matrix.py b/ci/ci_matrix.py new file mode 100755 index 0000000000..7e9f0ba649 --- /dev/null +++ b/ci/ci_matrix.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""The job matrices of build.yml and extended_checks.yml, one place, as JSON. + + ci_matrix.py build -> {"include": [cell, ...]} + ci_matrix.py extended -> {"include": [cell, ...]} + +A per-PR job fits a 35-minute wall or its cells run on the nightly cron (and on a manual +dispatch) instead. The workflow's pre_job step evaluates this script and the fan-out job reads +`fromJSON(needs.pre_job.outputs.)`, so the cells are data here rather than an include +list the runner merges by its own rules. `ci/test_ci_matrix.py` pins both sets. +""" +import json +import sys + +NIGHTLY_EVENTS = ("schedule", "workflow_dispatch") + + +def is_nightly(event_name): + return event_name in NIGHTLY_EVENTS + + +def _cells(target, architecture, presets, **props): + return [dict(target=target, architecture=architecture, cmake_preset=preset, sanitizers="none", **props) + for preset in presets] + + +def build_cells(event_name): + """build.yml: Release + Debug on every platform per PR; the sanitizer cells and windows + 64 Debug are nightly-only (each is a 40-55 minute job whose signal a day's cadence serves).""" + cmake = dict(build_system="cmake", cmake_generator="Ninja") + cells = [] + cells += _cells("linux", 64, ["Debug", "Release"], release_target="linux", release_arch="x86_64", + runner="ubuntu-latest", archive_ext="tar.gz", **cmake) + cells += _cells("linux_arm", 64, ["Debug", "Release"], release_target="linux", release_arch="arm64", + runner="ubuntu-24.04-arm", archive_ext="tar.gz", **cmake) + cells += _cells("darwin15", "arm64", ["Debug", "Release"], release_target="darwin15", release_arch="arm64", + runner="macos-15", architecture_string="arm64", archive_ext="tar.gz", **cmake) + cells += _cells("darwin26", "arm64", ["Debug", "Release"], release_target="darwin26", release_arch="arm64", + runner="macos-26", architecture_string="arm64", archive_ext="tar.gz", **cmake) + # win32 Release is the 32-bit compile+test gate; win32 Debug outgrew its test budgets and is gone. + cells += _cells("windows", 32, ["Release"], release_target="windows", release_arch="x86", + runner="windows-latest", architecture_string="Win32", archive_ext="zip", **cmake) + # The Win64 Release lane is the fast MSVC compile/interpreter gate: LLVM off, JIT off (the + # nightly build_windows_release_llvm_nightly job keeps the JIT sweep). Win64 Debug skips JIT too. + cells += _cells("windows", 64, ["Release"], release_target="windows", release_arch="x86_64", + runner="windows-latest", architecture_string="x64", archive_ext="zip", + llvm_disabled="ON", jit_disabled="ON", **cmake) + if is_nightly(event_name): + cells += _cells("windows", 64, ["Debug"], release_target="windows", release_arch="x86_64", + runner="windows-latest", architecture_string="x64", archive_ext="zip", + jit_disabled="ON", nightly_only="ON", **cmake) + for san in ("asan", "tsan", "ubsan"): + cells.append(dict(target="linux", architecture=64, cmake_preset="Release", sanitizers=san, + runner="ubuntu-latest", build_name="linux_" + san, nightly_only="ON", **cmake)) + return cells + + +def extended_cells(event_name): + """extended_checks.yml: per PR two darwin15 jobs split by role (core: the tree's own gates and + utils; modules: the module and service suites); the nightly runs every step on linux, darwin15 + and windows in one job each (role all).""" + cmake = dict(build_system="cmake", cmake_generator="Ninja") + darwin = dict(target="darwin15", architecture="arm64", runner="macos-15", architecture_string="arm64", **cmake) + if is_nightly(event_name): + return [ + dict(target="linux", architecture=64, role="all", runner="ubuntu-latest", **cmake), + dict(role="all", **darwin), + dict(target="windows", architecture=64, role="all", runner="windows-latest", architecture_string="x64", **cmake), + ] + return [dict(role="core", **darwin), dict(role="modules", **darwin)] + + +def main(argv): + if len(argv) != 3 or argv[1] not in ("build", "extended"): + sys.stderr.write(__doc__) + return 2 + cells = build_cells(argv[2]) if argv[1] == "build" else extended_cells(argv[2]) + print(json.dumps({"include": cells}, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/ci/test_ci_matrix.py b/ci/test_ci_matrix.py new file mode 100644 index 0000000000..b038c94756 --- /dev/null +++ b/ci/test_ci_matrix.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Pins the per-PR and nightly job matrices ci_matrix.py emits, and the workflow shapes that +consume them.""" +import json +import os +import re +import subprocess +import sys +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import ci_matrix # noqa: E402 + +WORKFLOWS = os.path.join(os.path.dirname(HERE), ".github", "workflows") + + +def names(cells): + return sorted("%s-%s-%s-%s" % (c["target"], c["architecture"], c["cmake_preset"], c["sanitizers"]) for c in cells) + + +PR_BUILD = sorted([ + "linux-64-Debug-none", "linux-64-Release-none", + "linux_arm-64-Debug-none", "linux_arm-64-Release-none", + "darwin15-arm64-Debug-none", "darwin15-arm64-Release-none", + "darwin26-arm64-Debug-none", "darwin26-arm64-Release-none", + "windows-32-Release-none", "windows-64-Release-none", +]) +NIGHTLY_ONLY_BUILD = sorted([ + "windows-64-Debug-none", "linux-64-Release-asan", "linux-64-Release-tsan", "linux-64-Release-ubsan", +]) + + +class BuildMatrix(unittest.TestCase): + def test_per_pr_cells(self): + for event in ("pull_request", "push"): + self.assertEqual(names(ci_matrix.build_cells(event)), PR_BUILD, event) + + def test_nightly_adds_exactly_the_slow_cells(self): + for event in ("schedule", "workflow_dispatch"): + self.assertEqual(names(ci_matrix.build_cells(event)), sorted(PR_BUILD + NIGHTLY_ONLY_BUILD), event) + + def test_nightly_only_cells_are_marked(self): + marked = names(c for c in ci_matrix.build_cells("schedule") if c.get("nightly_only") == "ON") + self.assertEqual(marked, NIGHTLY_ONLY_BUILD) + self.assertFalse(any(c.get("nightly_only") for c in ci_matrix.build_cells("pull_request"))) + + def test_every_cell_names_a_runner_and_a_generator(self): + for cell in ci_matrix.build_cells("schedule"): + self.assertTrue(cell.get("runner"), cell) + self.assertEqual(cell.get("cmake_generator"), "Ninja", cell) + self.assertEqual(cell.get("build_system"), "cmake", cell) + + def test_windows_release_64_is_the_llvm_free_gate(self): + cell = [c for c in ci_matrix.build_cells("pull_request") + if c["target"] == "windows" and c["architecture"] == 64 and c["cmake_preset"] == "Release"][0] + self.assertEqual((cell["llvm_disabled"], cell["jit_disabled"]), ("ON", "ON")) + + def test_release_cells_carry_archive_fields(self): + for cell in ci_matrix.build_cells("pull_request"): + self.assertIn("release_target", cell, cell) + self.assertIn("release_arch", cell, cell) + self.assertIn("archive_ext", cell, cell) + + def test_sanitizer_cells_carry_no_archive_fields(self): + for cell in ci_matrix.build_cells("schedule"): + if cell["sanitizers"] != "none": + self.assertNotIn("release_target", cell, cell) + self.assertEqual(cell["build_name"], "linux_" + cell["sanitizers"]) + + +class ExtendedMatrix(unittest.TestCase): + def test_per_pr_is_two_darwin_roles(self): + for event in ("pull_request", "push"): + cells = ci_matrix.extended_cells(event) + self.assertEqual([(c["target"], c["role"]) for c in cells], [("darwin15", "core"), ("darwin15", "modules")], event) + + def test_nightly_is_one_full_job_per_platform(self): + for event in ("schedule", "workflow_dispatch"): + cells = ci_matrix.extended_cells(event) + self.assertEqual([(c["target"], c["role"]) for c in cells], + [("linux", "all"), ("darwin15", "all"), ("windows", "all")], event) + + def test_every_cell_names_a_runner(self): + for event in ("pull_request", "schedule"): + for cell in ci_matrix.extended_cells(event): + self.assertTrue(cell.get("runner"), cell) + self.assertIn("architecture", cell, cell) + + +class WorkflowShapes(unittest.TestCase): + """The workflows consume the matrices through pre_job; the role split must keep the nightly + `all` role running every step.""" + + def read(self, name): + with open(os.path.join(WORKFLOWS, name), encoding="utf-8") as f: + return f.read() + + def test_both_workflows_read_their_matrix_from_pre_job(self): + for name, kind in (("build.yml", "build"), ("extended_checks.yml", "extended")): + text = self.read(name) + self.assertIn("matrix: ${{ fromJSON(needs.pre_job.outputs.matrix) }}", text, name) + self.assertIn("ci/ci_matrix.py %s" % kind, text, name) + + def test_role_conditions_exclude_one_role_only(self): + # `matrix.role == 'core'` would drop the step from the nightly `all` job; the only admitted + # spelling names the role a step does NOT run in + text = self.read("extended_checks.yml") + refs = re.findall(r"matrix\.role\s*(==|!=)\s*'([a-z]+)'", text) + self.assertTrue(refs, "extended_checks.yml carries no role conditions") + for op, role in refs: + self.assertEqual(op, "!=", "matrix.role %s '%s'" % (op, role)) + self.assertIn(role, ("core", "modules"), "matrix.role %s '%s'" % (op, role)) + + def test_nightly_only_build_cells_save_no_sccache_slot(self): + text = self.read("build.yml") + self.assertIn("if: github.ref == 'refs/heads/master' && matrix.nightly_only != 'ON'", text) + + # the steps that run only on the nightly cron, pinned by name: moving another step off the per-PR path is + # a deliberate edit here, with its preflight mirror or platform reason stated in the workflow + NIGHTLY_ONLY_STEPS = { + "Run examples from modules", + "Run tutorial dry-runs", + "Verify authored-doc code blocks (nightly only)", + "Cross-compile nano for cortex-m4", + "Compile tests/ with --ast-verify-batch", + "Coverage", + } + + def step_conditions(self, text): + """{step name: its first `if:` line} for every named step of a workflow.""" + conditions, current = {}, None + for line in text.splitlines(): + m = re.match(r'\s*- name: "(.*)"\s*$', line) + if m: + current = m.group(1) + conditions.setdefault(current, "") + continue + m = re.match(r"\s*if: (.*)$", line) + if m and current is not None and not conditions[current]: + conditions[current] = m.group(1) + return conditions + + def test_nightly_only_steps_are_exactly_the_pinned_set(self): + conditions = self.step_conditions(self.read("extended_checks.yml")) + nightly = {name for name, cond in conditions.items() if "github.event_name == 'schedule'" in cond} + self.assertEqual(nightly, self.NIGHTLY_ONLY_STEPS) + + +class CommandLine(unittest.TestCase): + def run_tool(self, *args): + tool = os.path.join(HERE, "ci_matrix.py") + return subprocess.run([sys.executable, tool, *args], capture_output=True, text=True) + + def test_emits_one_json_line_the_workflow_can_fromjson(self): + for kind in ("build", "extended"): + out = self.run_tool(kind, "pull_request") + self.assertEqual(out.returncode, 0, out.stderr) + self.assertEqual(out.stdout.count("\n"), 1) + self.assertIn("include", json.loads(out.stdout)) + + def test_rejects_an_unknown_matrix(self): + self.assertEqual(self.run_tool("release", "push").returncode, 2) + + def test_rejects_a_missing_event(self): + self.assertEqual(self.run_tool("build").returncode, 2) + + def test_each_kind_emits_its_own_cell_shape(self): + build = json.loads(self.run_tool("build", "pull_request").stdout)["include"] + extended = json.loads(self.run_tool("extended", "pull_request").stdout)["include"] + self.assertTrue(all("sanitizers" in c and "role" not in c for c in build)) + self.assertTrue(all("role" in c and "sanitizers" not in c for c in extended)) + + +if __name__ == "__main__": + unittest.main() diff --git a/include/daScript/misc/env_cfg.h b/include/daScript/misc/env_cfg.h index acf593a49e..8c8063ce18 100644 --- a/include/daScript/misc/env_cfg.h +++ b/include/daScript/misc/env_cfg.h @@ -32,6 +32,7 @@ namespace das { DAS_API const char * get_dasenv_jobque_team_eager_exit (); DAS_API const char * get_dasenv_team_prof (); DAS_API const char * get_dasenv_trace_module_load (); + DAS_API const char * get_dasenv_module_cache_limit (); // ambient variables daslang reads but does not own DAS_API const char * get_columns (); diff --git a/include/daScript/simulate/ARCHITECTURE.md b/include/daScript/simulate/ARCHITECTURE.md index 6b6932976e..d75b96756f 100644 --- a/include/daScript/simulate/ARCHITECTURE.md +++ b/include/daScript/simulate/ARCHITECTURE.md @@ -11,6 +11,31 @@ every program on every step. Amortized container work - growth in `src/simulate/ (repo root) / `runtime_table.h`, reached from eval nodes but running once per capacity change - is outside the hot set; its cost is judged against the allocate/copy/rehash it rides. +## Table key hashing + +A table key hashes on every rail as the interpreter's table node hashes it. A builtin key type - +`heap.h`'s `makeTableKeyValueNode` list: scalars, the vectors, the ranges, strings, pointers - +hashes as itself, `hash_function(context, key)`: `hash_uint32`/`hash_uint64` for the scalar and +range specializations, `hash_blockz64` for strings, `hash_block64` over `sizeof(key)` bytes +otherwise (8 for an `int2`, 12 for a `float3`). A handled key - a `ManagedValueAnnotation` +such as `Time` or a module's id type - hashes as its workhorse, `WrapType::type`, because +that is the das value type the annotation declares (`makeValueType`) and what the compiler casts +the key to before the node hashes it; `EntityId` hashes as an `int32`, `BigEntityId` as a +`vec4f`. The interpreter's table nodes (`runtime_table_nodes.h`), the JIT helpers +(`src/builtin/module_jit.cpp`, repo root), the JSON scanner (`src/simulate/json_scan.cpp`, repo +root), rtti and the C API call `hash_function` on the key type they hold. `KeyHash` +(`runtime_table.h`) is the same hash for the callers that hold a key of a C++ type - AOT's +`TTable`, the `__builtin_table_*` templates in `aot.h`, and the rehash a grow performs on every +stored non-string key (a string table reuses its stored hashes): it takes the workhorse detour +for a handled type and hashes a builtin type as itself, telling them apart by +`WrapsBuiltinValue` (`cast.h`, beside `WrapType`'s primary; a translation unit that sees the +vec4f detour from `jit_abi.h` sees the marks too, and one that sees neither hashes raw either way). The detour on a +builtin vector or range would hash a `vec4f`'s 16 bytes against the node's 8 or 12, and the raw +bytes of a handled type narrower than its workhorse (an 8-byte `ImVec2` wrapped to `vec4f`) +would miss the node's 16; either way every such key moves to another bucket at the first grow. +Non-string tables are open-addressed from their first slot (only string keys pack linearly up +to 8), so a disagreement shows on a one-key table as much as on a large one. + ## Sanctioned hot-path additions The ledger the checklist's hot-path rule routes to. Each entry: what was added, where, why diff --git a/include/daScript/simulate/REVIEW.md b/include/daScript/simulate/REVIEW.md index 8b38183d23..d408bf4545 100644 --- a/include/daScript/simulate/REVIEW.md +++ b/include/daScript/simulate/REVIEW.md @@ -14,6 +14,15 @@ checklist on its own. that refuses a record written under other policies, so a field missing from it is a policy the cache silently ignores. +- **A diff that hashes a table key hashes a builtin key type - one in `heap.h`'s + `makeTableKeyValueNode` list - as itself through `hash_function(context, key)` (`hash.h`), + and a handled key type as the value type its annotation's `makeValueType()` returns.** A + table grow rehashes every non-string key with `KeyHash` (`runtime_table.h`), so a site that + hashes a key type differently loses every key of that type at the first grow. + +- **A diff that changes `KeyHash` (`runtime_table.h`) or `WrapsBuiltinValue` (`cast.h`) states + in its own PR description which key types change hash value.** + - **A diff that makes the hot path more expensive per evaluated expression is a defect - an added load, branch, call, copy, or counter, a direct call becoming indirect, a static dispatch becoming virtual, and an unboxed value becoming a boxed round-trip all count.** @@ -22,7 +31,8 @@ checklist on its own. `invoke` / `invokeEx` (`simulate.h`), or an AOT-side function or template under this folder that generated code executes per evaluated expression. Such a diff - including one an optimized build flattens to nothing - lands its entry under `ARCHITECTURE.md`'s - sanctioned hot-path additions in the same diff; that ledger defines what the entry says. + sanctioned hot-path additions in the same diff: what was added, where, why correctness + required it, and the alternative that was rejected. - **A diff that changes the layout of a `debug_info.h` struct - a field added, removed, reordered, or retyped, or a base changed - states a per-consumer verdict (updated / no diff --git a/include/daScript/simulate/cast.h b/include/daScript/simulate/cast.h index c6bfaf0c56..475b7482dc 100644 --- a/include/daScript/simulate/cast.h +++ b/include/daScript/simulate/cast.h @@ -8,6 +8,8 @@ namespace das { template struct WrapType { enum { value = false }; typedef TT type; typedef TT rettype; }; + // a builtin das value type the interpreter's table nodes hash as itself; a handled type hashes as its workhorse + template struct WrapsBuiltinValue { enum { value = false }; }; template struct JitConstRefByValue { enum { value = false }; }; template struct WrapArgType { typedef TT type; }; template struct WrapRetType { typedef TT type; }; @@ -467,6 +469,22 @@ namespace das template <> struct cast : cast_iVec {}; template <> struct cast : cast_iVec {}; + // the builtin value types the interpreter's table nodes hash as themselves, beside the primary + // so no include order can see WrapType's vec4f detour (jit_abi.h) without this + template <> struct WrapsBuiltinValue { enum { value = true }; }; + template <> struct WrapsBuiltinValue { enum { value = true }; }; + template <> struct WrapsBuiltinValue { enum { value = true }; }; + template <> struct WrapsBuiltinValue { enum { value = true }; }; + template <> struct WrapsBuiltinValue { enum { value = true }; }; + template <> struct WrapsBuiltinValue { enum { value = true }; }; + template <> struct WrapsBuiltinValue { enum { value = true }; }; + template <> struct WrapsBuiltinValue { enum { value = true }; }; + template <> struct WrapsBuiltinValue { enum { value = true }; }; + template <> struct WrapsBuiltinValue { enum { value = true }; }; + template <> struct WrapsBuiltinValue { enum { value = true }; }; + template <> struct WrapsBuiltinValue { enum { value = true }; }; + template <> struct WrapsBuiltinValue { enum { value = true }; }; + // 16/8-bit lattice vectors — byte-packed in the low bytes of the slot; prune handles // every size (incl. the odd 2/3/6-byte widths) after the generic-memcpy branch above template <> struct cast : cast_fVec {}; diff --git a/include/daScript/simulate/runtime_table.h b/include/daScript/simulate/runtime_table.h index aa378697be..8f9e60108b 100644 --- a/include/daScript/simulate/runtime_table.h +++ b/include/daScript/simulate/runtime_table.h @@ -12,11 +12,12 @@ namespace das DAS_API extern const char * rts_null; + // what the interpreter's table node hashes for this key type - ARCHITECTURE.md, "Table key hashing" template struct KeyHash { __forceinline uint64_t operator () ( Context & context, const KeyType & key ) { using workhorse = typename WrapType::type; - if constexpr ( is_same::value ) { + if constexpr ( is_same::value || WrapsBuiltinValue::value ) { return hash_function(context, key); } else { return hash_function(context, cast::to(cast::from(key))); diff --git a/plans/ci_preflight_budget.md b/plans/ci_preflight_budget.md new file mode 100644 index 0000000000..b53c677043 --- /dev/null +++ b/plans/ci_preflight_budget.md @@ -0,0 +1,107 @@ +# The pre-merge budget: preflight in 20 minutes, CI in 35 per job + +Ruling: on the M5 box `preflight --full` fits in 20 minutes or a gate is not in preflight; a per-PR CI job +fits in 35 minutes or its steps go to the nightly run. What does not fit is retired, not tolerated. + +## Baseline (measured 2026-09-04, module-cache follow-up branch) + +Local chain (`make-pr` then `preflight --full`), serial on 16 cores: + +| Phase | Wall | Composition | +|---|---|---| +| make-pr gates | 12.5 min | sync, review-md walk, stamp-reach, dupes (the bulk), ast-verify, jit-smoke | +| preflight, non-dasLLAMA | ~20 min | tests-jit ~6.5, sphinx docs ~3.3, AOT build+run ~2 (7 cold), interp ~1.7, imgui ~1.4, sequence <1, the fast gates ~3 | +| dasllama-model-free | ~6 min | 63 files, warm caches; ~10 s/file is engine compile the module cache does not reach | +| dasllama-stocked | ~25 min | model wall: test_batch_decode 263 s, test_ple_modes ~560 s, gemma3v 109 s, vision_chat ~110 s | + +CI, last master run (9836bcb91): extended_checks linux 75 min (build 23.7, examples 8.4, tutorials 7.8, +dasllama-server 6.9, utils tests 4.2, coverage 3.5, ser/deser 2.8, static 2.8, facade lint 2.3, MCP 2.3), +extended_checks darwin15 46 min (build 16.3, tutorials 10.9, examples 5.6); build lane 55 min (asan 55, windows +Debug 52, tsan 51, windows Release 50, linux Release 46, ubsan 45, bundle_smoke 41 - the build step is the +cost: linux Release 1302 objects in 30 min, cold); CodeQL 42.5. sccache: `SCCACHE_CACHE_SIZE=500M` while one +Release build is 718 MB of objects, so no slot ever holds a build (linux: cold every run; windows: 22% hits); +the repo holds 17 GB of caches, 4.3 GB of it ten per-commit CodeQL databases; the limit is ~25 GB. + +Structural causes: everything serial; review-md and ast-verify run in both halves of the chain; no reach +model for module-owned gates; dupes (advisory) on the critical path; the dasLLAMA per-file engine compile. + +## Preflight (utils/internal/preflight) + +Tiers, printed by `--list-gates`: + +- **fast** (serial, first; a red stops the run before the lanes): untracked, format, lint, ast-verify, + cpp-syntax, review-md, md-ascii, hash-refs, dasgen, ci-das, compile-sweep (every program root under + `utils/`, `examples/`, `tutorials/`, the modules' examples and utils, `-compile-only`, parallel - 995 roots + in 18 s here). +- **lane** (`--full`, run concurrently, wall = the longest): tests-cpp, tests-interp, tests-jit, tests-aot, + docs, utils-tests (`run_utils_tests`, ~100 s here). +- **module** (never in `--full`; `--only ` when working on the module): imgui, sequence, + dasllama-model-free, dasllama-stocked. +- **reach**: a gate names the path prefixes that reach it; `src/`, `include/`, `daslib/`, `dastest/`, + `CMakeLists.txt` reach everything. A diff that misses a gate's reach skips it with the reason. +- make-pr's default chain drops review-md and ast-verify (preflight owns them) and dupes (an advisory + report: `make-pr --only dupes` when wanted); it keeps sync, stamp-reach, jit-smoke, then chains preflight. + +Acceptance: a full run with a core change under 20 minutes on the M5 box; a dasLLAMA-only diff in about 5. + +Measured (2026-09-04, this branch, a core diff of 10 das / 3 cpp files): `preflight --full` 11.5 min wall. +Fast tier 3.1 min serial (compile-sweep 108 s - 730 light roots in an 18-wide pool 66 s + 25 engine roots +serial through one module cache 42 s; lint 24; cpp-syntax 18 - a header change sweeps 203 TUs; ci-das 12; +format 6; review-md 7). Lanes concurrent, wall = tests-aot 513 s (test_aot build 402 + run 111); docs 336 +(sphinx-html 297); utils-tests 186; tests-jit 184; tests-interp 116; tests-cpp 20. The pool width lesson: +`get_total_hw_threads()` is the jobque's worker count (5 on the 18-core M5), so a pool sized by it runs 5 +wide; size pools by `get_total_hw_cores()`. + +## CI (.github/workflows) + +- The matrices are data: `ci/ci_matrix.py build|extended ` emits the cells, `pre_job` evaluates it, + the fan-out job reads `fromJSON(needs.pre_job.outputs.matrix)`; `ci/test_ci_matrix.py` pins both sets + and the role-condition spelling (`.github/workflows/REVIEW.md`). +- sccache: build.yml's slots were never capped (the default 10G; the compressed tarballs run 104 MB to + 971 MB) - the 500M cap was extended_checks' alone, and it is now 1500M so the nightly save holds a + complete object set. The nightly-only cells (sanitizers, windows 64 Debug) save no slot: nothing + restores one (frees ~1.3 GB). CodeQL stays per PR: 20 min on the last PR, inside the budget, and no + local gate mirrors it (its ~430 MB per-commit databases stay; the repo's caches total ~12 GB of 25). +- extended_checks per PR = two darwin15 jobs, `core` and `modules` (the step lists: `skills/internal/preflight.md` + sec."extended_checks.yml"). Estimated 20-26 min each at a warm sccache. linux and windows extended_checks, + tutorial dry-runs, the run form of examples, coverage, nano cross-compile: nightly (same workflow, + `event_name == 'schedule' || 'workflow_dispatch'`, role `all`). daslang_static stays per PR (modules role): + a step leaves the per-PR path only when a preflight gate mirrors it, and nothing local sweeps the static + binary. `ci/test_ci_matrix.py` runs in the core role and as preflight's `ci-matrix` gate. Two darwin lane steps that were + linux-only needed portable shells: `mapfile` (bash 3.2 on the runner) became a read loop, `timeout` + (no coreutils) became perl's `alarm`. +- build lane per PR: Release + Debug on linux/darwin/windows/linux_arm as today minus asan/tsan/ubsan and + windows 64 Debug, which are nightly-only cells. Every remaining job is a build step. + +Acceptance: every per-PR job under 35 minutes on the first PR after the change (measure with the Actions API: +run, job, step walls; the script in the session scratchpad becomes `utils/internal/ci-timing/` if kept). + +## Ledger - not done in the preflight arc + +- Preflight's orchestration arms have no unit test: `run_lanes` (the empty-report arm, the tag map, a + short child list), `collect_changed_paths`, the reach-skip and lane-deferral arms of `main`, the gate + functions' skip/fail arms (`gate_ci_matrix`, `gate_utils_tests`, `gate_tests_aot`'s two details), the + sweep workers' bounds arms. The pure halves (`tier_runs`, `sweep_excluded`, `is_sweep_root_text`, + `is_heavy_sweep_root`, `failed_files_block`, `parse_gate_reports`, `reach_hit`, `is_sweep_root_path`) + live in `config.das` with `tests/test_changed_set.das`; the arms above need a spawned-child harness + (`tests/dastest/test_preflight_config.das` is the precedent). +- Gate candidates the audits named: a `.github/workflows/REVIEW.das` that fails `continue-on-error` or + a trailing `|| true` on any per-PR job's step; `examples/games/REVIEW.das` reporting a `pages.yml` line + that names two or more game ids outside a `for g in` loop; `include/daScript/simulate/REVIEW.das` + pinning the set of table-key hashing sites (five of eight live outside that folder, so the checklist + never opens for them) - or routing every site through `KeyHash` so the rule retires. +- `compute_worker_count(n, 0)` in `utils/common/parallel_workers.das` defaults to `get_total_hw_threads()`, + the jobque's worker count (5 on the 18-core M5): the cpp-syntax sweep and detect-dupe run 5 wide here. +- Two tracked generated files come back modified after every cmake build (same bytes, another line + layout): `modules/dasUnitTest/unit_test.das.inc`, `tutorials/integration/cpp/class_adapters_module.das.inc`. + +## dasLLAMA long tests (after the above) + +- Compile dressed as a test: `test_exe_smoke` (108 s, builds an exe), `test_tok_seed` (90 s) and + `test_parity_pregate` (91 s) require `lcpp_bench` by path and pay its engine compile for a header + parser and a pregate - the helpers move into a small module the bench requires. +- Model wall, per file: `test_batch_decode` 263 s, `test_ple_modes` ~560 s, `test_gemma3v` 109 s, + `test_vision_chat` ~110 s, `test_kquant` 82 s, `test_whisper` 80 s, `test_parity` 40 s - fewer carriers, + one load per file, smaller fixtures. +- The engine compile floor: the module cache surviving into dastest's runtime `compile_file` (per test + file key; an engine record is ~210 MB - why, first). diff --git a/skills/internal/build_and_debug.md b/skills/internal/build_and_debug.md index fc03e4f009..6393f22bda 100644 --- a/skills/internal/build_and_debug.md +++ b/skills/internal/build_and_debug.md @@ -95,7 +95,7 @@ Dev-tier rails that cut the edit-compile-run loop. Never benchmark through `--ji ## Front-end (AST) module cache -`daslang` and `daslang-live` cache the compiled AST module graph through the env-serializer rail, silently, at `.jitted_scripts/module_cache/-.dascache` relative to the cwd. The first run writes the file; later runs deserialize the post-infer modules instead of parsing them - roughly an order of magnitude on a large module graph, measured with an explicit `-module-cache`. Benchmarking a number that includes compile time? Pass `-no-module-cache`. +`daslang` and `daslang-live` cache the compiled AST module graph through the env-serializer rail, silently, at `.jitted_scripts/module_cache/-.dascache` relative to the cwd. The first run writes the file; later runs deserialize the post-infer modules instead of parsing them - roughly an order of magnitude on a large module graph, measured with an explicit `-module-cache`. Benchmarking a number that includes compile time? Pass `-no-module-cache`. The directory is capped at `DAS_MODULE_CACHE_LIMIT` megabytes (4096 by default; an engine root's record is ~200 MB): after each write the oldest records by mtime go, a record a run read counts as fresh, and an explicit `-module-cache` path is never pruned. - **The hash keys the compile, not just the script:** the normalized script path; the host's command line up to `--` (`-v1syntax`, `-jit`, `-project` are different compiles; the script's own arguments after `--` are not); every `DAS*` environment variable (macros read the tune/JIT environment at compile time); and the running daslang binary's mtime and size, resolved through the OS rather than read off `argv[0]`. Same-named roots, a changed flag set, a changed environment and a rebuild each get their own file; a launch by bare name through `PATH` keys the same file as a launch by path. - **Off:** `-no-module-cache` turns it off and beats an explicit `-module-cache ` on the same command line, so a tool that spawns `daslang` appends `-no-module-cache` as an override. Beside `-ser`/`-deser` the binary rejects the command line instead. It is off on its own under `-exe` (the exe is parsed as one compilation unit on purpose, so the optimizer sees the whole graph), `-compile-only` (preflight and lint spawn one daslang process per file, each on a different root, so no run is ever warm), `-documentation` (a one-shot generator - there is no second run for a cache to serve), `-use-aot` (an AOT-consuming run links the compiled functions against the AOT stubs by hash; it is an artifact run like `-exe`, and the link sees exactly the compile the stubs were generated from) and `--das-wait-debugger` (the serializer refuses a program that requires the debugger, so the cache would only print that refusal). diff --git a/skills/internal/environment_variables.md b/skills/internal/environment_variables.md index 8425e1310c..e240fd903c 100644 --- a/skills/internal/environment_variables.md +++ b/skills/internal/environment_variables.md @@ -60,6 +60,12 @@ the log. config change, and it reverts the moment the variable goes away. Surrounding whitespace is ignored and unknown codes are harmless. The `-no-lint` command-line flag skips the lint pass entirely. +## Module cache + +| Variable | Type | Effect | +|---|---|---| +| `DAS_MODULE_CACHE_LIMIT` | number (MB) | Size cap of the default module-cache directory (`.jitted_scripts/module_cache/`, `skills/internal/build_and_debug.md`). After a run writes a record, the oldest records by mtime go until the directory fits; a record a run read counts as fresh. Default 4096; `0` turns eviction off; garbage keeps the default. An explicit `-module-cache ` is never pruned. The only `DAS*` variable the record key leaves out. | + ## Diagnostics | Variable | Type | Effect | diff --git a/skills/internal/make_pr.md b/skills/internal/make_pr.md index b1b86cb98a..c7b1844501 100644 --- a/skills/internal/make_pr.md +++ b/skills/internal/make_pr.md @@ -3,12 +3,14 @@ Complete every step in order; fix a failure before proceeding. **The mechanical gates are one command:** `daslang utils/internal/make-pr/main.das --` runs -sync -> review-md walk -> stamp-reach -> dupes -> ast-verify -> jit-smoke (stamp-reach checks -the changed record stores' engine stamps are reachable from HEAD - a rebase orphans them; the -last two auto-skip when the diff has no macro/AST or JIT surface), then chains `preflight --full` (`skills/internal/preflight.md` -maps each gate to its CI lane). `--only ` reruns one; `--no-preflight` skips the chain; exit 2 names the red gate. This -file is the authority on fix policy and on what the tool prints as STILL YOURS. Commit, run -it, push once (one batched PR). +sync -> stamp-reach -> jit-smoke (stamp-reach checks the changed record stores' engine stamps +are reachable from HEAD - a rebase orphans them; jit-smoke auto-skips when the diff has no JIT +surface), then chains `preflight --full` (`skills/internal/preflight.md` maps each gate to its +CI lane; review-md and ast-verify are its fast-tier gates, one pass each). `--only ` +runs one gate by name - `--only dupes` for the duplicate-function report (advisory, triage is +yours, off the default chain), `--only review-md` / `--only ast-verify` for a rerun; `--no-preflight` +skips the chain; exit 2 names the red gate. This file is the authority on fix policy and on what +the tool prints as STILL YOURS. Commit, run it, push once (one batched PR). **The full preflight runs ONCE per PR - never a second full run.** On failure fix everything, validate each fix with the **targeted** gate or an isolated repro (`--only `, the @@ -27,14 +29,14 @@ kills the chain's JIT loads, AOT links, and spawned tools mid-suite). | 0 Sync | make-pr `sync` | Red = behind origin/master: rebase (never onto local `master`), re-run. A listed PR-set file you did not edit = the rebase went wrong; a conflict on a file also changed upstream keeps origin/master's. Squash only AFTER the rebase - `git reset --soft master` on a stale `master` bakes other PRs in; already pushed: rebase + `git push --force-with-lease`. Re-read any `skills/*.md` / `REVIEW*.md` the rebase changed | | 0 Untracked | preflight `untracked` gate | Empty at PR time - commit, delete, or ignore each (`.gitignore`; `.git/info/exclude` for box-local) | | 0a0 Comment harvest | the diff's ADDED comments, per touched module root | Working comments are welcome while building the PR; this row is where they settle. When the diff adds comments beyond the hygiene skill's kept set, spawn ONE `harvester` per touched module root, scoped to the comments the diff adds (a full-file harvest is the on-first-touch sweep, not a PR gate). YOU rule on its ledger - RENAME first (the strongest resolution), RULE proposals land in the folder's REVIEW.md, a FACT lands as three things - the section, its `{#anchor}`, and the `[arch]` citation from the function the comment came from (the harvest duty in REVIEW_COMMON.md; the ledger names the citer), KEEP one-liners are `//!` contract comments, TODO signals are possibly unfinished PR work, lint candidates surface per CLAUDE.md's lint-opportunities rule. Sits before the audits because landings re-enter the diff | -| 0a REVIEW audit | make-pr `review-md` | Red = a discovered `REVIEW.das` gate failed - fail-fix, no agents until green. Then one `review-md-auditor` per checklist. Discovered rules bind on top of this file; checklist defects fixed in the same batch | +| 0a REVIEW audit | preflight `review-md` gate (`make-pr --only review-md` reruns it alone) | Red = a discovered `REVIEW.das` gate failed - fail-fix, no agents until green. Then one `review-md-auditor` per checklist. Discovered rules bind on top of this file; checklist defects fixed in the same batch | | 0a TDD audit | one `tdd-auditor` over the whole diff, REVIEW.md folders or not (`skills/tdd_audit.md`) | UNTESTED branch -> test in the same change, never a follow-up promise. UNPROVEN -> run its named settling gate or state the claim in the PR body. RETUNED/WEAKENED test edit -> restore the expectation/instrument or state the reason | | 0a2 Style hygiene | `style-hygiene-auditor` (`skills/comment_style_hygiene.md`) | Mandatory run, non-blocking findings: fix each or consciously decline it | | 0a3 Woodpecker | external codex round (`skills/internal/woodpecker.md`), background Bash at final branch shape; keep working the checklist while it runs | Every arc, trivial or not; non-trivial arcs re-round on the fixed tip. Verify each finding; harvest before step 6 | | 0b Build drift | nuke `build/` only on the three symptoms below | No proactive clean build. Never run full preflight on a Debug host | | 1 Lint | preflight lint gate (debug one file: MCP `lint`) | **Zero warnings** - CI runs the same utility on the changed set. Fix it (idiom table in `CLAUDE.md`), or `// nolint:CODE` **with the reason**, for a known false positive only (handled types like `xml_node` need `var`) | -| 1.5 Dupes | make-pr `dupes` (scoping + report; modes: `skills/internal/detect_dupe.md`) | Triage is yours: reuse an exact match, justify the sibling, or extract a helper. Widen the corpus by `daslib` for a new generic helper. Skip for tests/fixtures/generated-only PRs | -| 1.6 AST verify | make-pr `ast-verify` | **Zero** `AST verify` lines and no crash - the tree is clean tree-wide, so any report is a bug in what built the node. A compile error is not a failure (many tests assert one). Plain `--ast-verify` on ONE file names the pass that broke it (`skills/das_macros.md`) | +| 1.5 Dupes | `make-pr --only dupes` (scoping + report; modes: `skills/internal/detect_dupe.md`) - off the default chain, run it once at final branch shape | Triage is yours: reuse an exact match, justify the sibling, or extract a helper. Widen the corpus by `daslib` for a new generic helper. Skip for tests/fixtures/generated-only PRs | +| 1.6 AST verify | preflight `ast-verify` gate (`make-pr --only ast-verify` reruns it alone) | **Zero** `AST verify` lines and no crash - the tree is clean tree-wide, so any report is a bug in what built the node. A compile error is not a failure (many tests assert one). Plain `--ast-verify` on ONE file names the pass that broke it (`skills/das_macros.md`) | | 1.7 Workarounds | `git diff origin/master..HEAD` - read every changed file | A smell (below) is a STOP-and-decide: surface fix-vs-workaround and **ask the user** | | 2 Tests | preflight tests gate (debug one file: MCP `run_test`) | Failures (assertions) and errors (compilation) both count. Fix yours and obvious pre-existing ones; **ask the user** about non-obvious - never call one pre-existing without checking the affected `tests/dasX/` against master's count. Changed `modules/X/daslib/`? Run that module's tests even if your build disables it (CI enables all of `ci/release_modules.txt`) | | 2.5 JIT smoke | make-pr `jit-smoke` | The smoke files run through `dastest -jit`; a verifier error or a failed run is red. Widen to `tests/soa/test_soa_basic.das` + `tests/language/typeAlias.das` for generic-instance or capture-frame changes. Windows `clang-cl` "program not executable" at the `.dll` link is linker discovery, not codegen - ignore it; end-to-end needs WSL (`skills/internal/wsl_ci_repro.md`) | diff --git a/skills/internal/preflight.md b/skills/internal/preflight.md index 30b8c1a2d9..c05ca38987 100644 --- a/skills/internal/preflight.md +++ b/skills/internal/preflight.md @@ -1,13 +1,23 @@ # Preflight - CI lane <-> local mirror -`daslang utils/internal/preflight/main.das` runs the fast tier (`--list-gates` -prints every gate and its tier). `-- --full` adds the expensive gates: the -untracked gate, dasgen freshness, the CI-only-das compile sweep, the doc gates, -ctest, the interp/JIT/AOT suites, the sequence smoke, the dasImgui suite, and - -when the diff touches `modules/dasLLAMA/` - the dasLLAMA `model-free` and -`stocked` suites. `--only ` / `--skip ` select subsets; -`--lint-skip-exe-rail` trims the lint gate to its interp rails. A gate whose -host tool or module is missing reports `SKIP` with an install/rebuild hint. +`daslang utils/internal/preflight/main.das` runs the **fast tier**: format, lint, +ast-verify, cpp-syntax, review-md, md-ascii, hash-refs, untracked, dasgen, ci-das, +ci-matrix (`python3 ci/test_ci_matrix.py`, when the diff touches `.github/` or `ci/`) +and compile-sweep (every program root under `utils/`, `examples/`, `tutorials/` +and the modules' examples and utils, compile-only, in parallel - the per-PR form +of CI's examples and tutorial runs), serially, and a red stops the run. `-- --full` +then runs the **lanes** at once - docs, tests-cpp, tests-interp, tests-jit, +tests-aot, utils-tests - so the wall is the longest lane (`--serial` for +diagnosis). **Module gates** never run from a tier: `--only imgui`, `--only +sequence`, `--only dasllama-model-free` when the module is the work. A gate +with a **reach set** skips, with the reason, when nothing under its paths or the +core (`src/`, `include/`, `daslib/`, `dastest/`, `CMakeLists.txt`, `cmake/`) +changed; `--only` runs a gate whatever changed. `--list-gates` prints tier, reach +and description; `--skip ` drops gates; `--lint-skip-exe-rail` trims the +lint gate to its interp rails. A gate whose host tool or module is missing +reports `SKIP` with an install/rebuild hint. The budget the tiers serve: a full +run fits 20 minutes on the M5 box, or the gate is not in preflight +(`plans/ci_preflight_budget.md`). Each gate line carries its breakdown indented underneath, on PASS as well as FAIL: the build/run split for a gate that builds before it sweeps (`tests-aot`, `sequence`, `imgui`), @@ -51,10 +61,12 @@ working-tree copy. | Workflow | Trigger | Jobs | |---|---|---| -| `build.yml` (per-PR) | every PR commit (`pull_request`) + pushes to `master` | `build` matrix (5 targets x Debug/Release/RelWithDebInfo x sanitizers), `bundle_smoke`, `build_linux_gcc` | -| `build.yml` (nightly) | `schedule` cron (daily 02:00 UTC) | `build_windows_mingw` + `build_windows_clangcl` (gated OFF per-PR) **plus the full build matrix, whose Release cells (sanitizers included) run the full AOT sweep** ("Slow Release Tests"). Breaks surface within ~24 h, not at PR time | +| `build.yml` (per-PR) | every PR commit (`pull_request`) + pushes to `master` | `build` matrix (`ci/ci_matrix.py build`: Debug + Release on linux, linux_arm, darwin15, darwin26; windows 32 Release, windows 64 Release), `bundle_smoke`, `build_linux_gcc` | +| `build.yml` (nightly) | `schedule` cron (daily 02:00 UTC) | `build_windows_mingw` + `build_windows_clangcl` (gated OFF per-PR) **plus the full build matrix - the per-PR cells, the sanitizer cells (linux Release asan/tsan/ubsan) and windows 64 Debug - whose Release cells run the full AOT sweep** ("Slow Release Tests"). Breaks surface within ~24 h, not at PR time | | `nightly_imgui.yml` | `schedule` cron (daily 03:00 UTC) + `workflow_dispatch` | dasImgui playwright suite on ubuntu + macos - section below | -| `extended_checks.yml` | every PR | linux + darwin15-arm64 + windows, ALL release modules ON | +| `extended_checks.yml` (per-PR) | every PR | two darwin15-arm64 jobs, `core` and `modules` (`ci/ci_matrix.py extended`), ALL release modules ON - section below | +| `extended_checks.yml` (nightly) | `schedule` cron (daily 04:00 UTC) + `workflow_dispatch` | one job each on linux, darwin15 and windows running every step (role `all`), including the ones too slow for a PR: tutorial dry-runs, the run form of examples, coverage, the nano cross-compile, the AST verify tree sweep, doc-verify | +| `codeql.yml` | every PR and `master` push touching `src/`, `include/`, `modules/`, `tests-cpp/` + a weekly cron | CodeQL over the C++ surface, ~20 min on a PR; no local mirror, so it stays per PR | | `wasm_build.yml` | every PR | emscripten build of `web/` on 3 OSes + `wasm_cross` | | `build_eastl.yml` | every PR | EASTL shadow-config build + no-fileio build (linux clang) | | `doc.yml` | only if `doc/**`, `daslib/**`, `src/builtin/**`, `modules/dasImgui/**`, `modules/dasVulkan/**`, or `modules/dasLLAMA/dasllama/**` changed | the doc gates | @@ -76,7 +88,7 @@ part of ALL) as a compile+link gate, and run no AOT tests. | AOT sweep (full) | `cmake --build build --config Release --target test_aot`, then `bin/Release/test_aot.exe -use-aot dastest/dastest.das -- --use-aot --color --failures-only --max-file-time 30 --timeout 1800 --test tests` | nightly + manual dispatch only, so this is the **only** pre-push gate for AOT regressions outside tests/language - don't skip it | | AOT subset gate | `cmake --build build --config Release --target test_aot_subset` (add `--target run_tests_aot_subset` to sweep tests/language too) | what per-PR lanes build | | Debug lanes | `cmake --build build --config Debug --target daslang`, then the sweep against `bin/Debug/daslang.exe` - Debug coexists in-checkout with Release (`bin/Debug/`, `_debug.shared_module`) | Debug bypasses the fused interpreter permutations: a fused-path-only fix passes Release everywhere and trips Debug, and fused-path bugs need Release. Touched `src/simulate/simulate_fusion_*`? run both | -| Sanitizer lanes (linux Release asan/tsan/ubsan) | WSL: `CC=clang CXX=clang++ cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DDAS_USE_SANITIZER=`, then the JIT sweep on `tests/language` | not mirrorable on Windows/mac. CI applies LSan suppressions (`format_error`, `uriParseSingleUriA`, `uriMakeOwner`) | +| Sanitizer lanes (linux Release asan/tsan/ubsan) - nightly | WSL: `CC=clang CXX=clang++ cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DDAS_USE_SANITIZER=`, then the JIT sweep on `tests/language` | not mirrorable on Windows/mac. CI applies LSan suppressions (`format_error`, `uriParseSingleUriA`, `uriMakeOwner`). Nightly + dispatch only (40-55 minute jobs); force one early with `gh workflow run build.yml` | | linux_arm / darwin lanes | mac: same commands as linux; from Windows not mirrorable | ARM reds (LLVM SelectionDAG, alignment) are CI-only signals | ## build.yml - bundle_smoke (linux) @@ -126,6 +138,22 @@ headers live there, and TUs like `src/simulate/fs_file_info.cpp` include them. ## extended_checks.yml +Per PR the workflow is two darwin15 jobs, `extended_checks (darwin15, core)` and +`extended_checks (darwin15, modules)`, each inside the 35-minute budget: `core` runs the +tree's own gates - dasgen, utils tests, standalone exes, formatter, lint, ast-verify, the +python gates, review-md discovery, dastest's own suite, the REVIEW.das gates, ci-das - +and `modules` the module and service suites - ser/deser, MCP tools, boulder-dash, +dasllama-server, env-knob registries, the dasLLVM vector-math rail, facade lint, +dasweb-playground, dasllama-ladder, dasweb-buildd, dasweb-verify, the sequence smoke, the +daslang_static sweep. A step's condition is `matrix.role != ''`, so the nightly job +(role `all`, one each on linux, darwin15, windows) runs every step plus the nightly-only ones: +tutorial dry-runs and the run form of examples (preflight's `compile-sweep` is their per-PR +mirror), the AST verify tree sweep and doc-verify (policy), coverage (a report, not a gate), +and the nano cross-compile - platform-bound: its arm-none-eabi toolchain is an apt package no +darwin cell and no developer box carries, so it has no per-PR cell and no local mirror. The cells are `ci/ci_matrix.py extended `; `ci/test_ci_matrix.py` +pins them and the condition spelling. The per-PR mirror of the nightly-only compile steps is +preflight's `compile-sweep` gate. + **CI configures with ALL release modules ON** - `ci/release_modules.txt` flips `DAS_HV/LLVM/AUDIO/PUGIXML/SQLITE/GLFW_DISABLED=OFF`. A local build with several OFF compiles none of the module-gated `.das` and C++ (dasOpenGL helpers, @@ -141,9 +169,9 @@ cmake -B build -DDAS_HV_DISABLED=OFF -DDAS_LLVM_DISABLED=OFF -DDAS_AUDIO_DISABLE |---|---|---| | Markdown ASCII gate | preflight's `md-ascii` gate (fast tier, runs when the diff touches any `.md`); manual: `python3 ci/fix_md_ascii.py --check`, fix in place by dropping `--check` | em-dashes/arrows/ellipses in new markdown are the usual trip | | dasgen freshness | ` utils/internal/dasgen/gen_bind.das` then `git diff --exit-code -- include/daScript/builtin/` | regen + commit if dirty; `skills/internal/visitor_gen_bind.md` | -| Run examples | `cmake --build build --config Release --target run_examples` | | -| Utils tests | `cmake --build build --config Release --target run_utils_tests` | | -| Tutorial dry-runs | `cmake --build build --config Release --target dry_run_tutorials` | compile rot in `tutorials/` after daslib API changes | +| Run examples - **nightly** | `cmake --build build --config Release --target run_examples`; per PR, preflight's `compile-sweep` gate compiles every example root | the run form is 5-8 minutes a lane | +| Utils tests | `cmake --build build --config Release --target run_utils_tests` - preflight's `utils-tests` lane | | +| Tutorial dry-runs - **nightly** | `cmake --build build --config Release --target dry_run_tutorials`; per PR, preflight's `compile-sweep` gate compiles every tutorial root | compile rot in `tutorials/` after daslib API changes; the run form is 8-11 minutes a lane | | Standalone exes | `cmake --build build --config Release --target all_utils_exe`, plus ` -exe -output bin/das-fmt utils/das-fmt/dasfmt.das` and `... bin/das-lint utils/lint/main.das` | `-exe` needs dasLLVM + lld-link on PATH | | Sequence smoke | Windows: `pwsh examples/games/sequence/ci_smoke_test.ps1 "$(pwd)"`; linux/mac: `bash examples/games/sequence/ci_smoke_test.sh "$(pwd)"` | build the runtime modules first: `cmake --build build --config Release --target dasModuleGlfw dasModuleLiveHost dasModuleHV dasModuleAudio dasModulePUGIXML dasModuleStbImage`. **The only pre-merge lane compiling GLFW-gated `.das` like dasOpenGL** - run it for type-system / daslib-generics changes | | Formatter `--verify` | preflight's `format` gate runs it exactly (tracked files via `--files-from`); manual: ` utils/das-fmt/dasfmt.das -- --path ./ --verify --exclude-mask build/` | CI's second verify pass uses an `-exe`-compiled `bin/das-fmt.exe`; the mask skips generated `.das` under the build dir (nightly doc-verify extracts RST snippets there) | @@ -151,13 +179,14 @@ cmake -B build -DDAS_HV_DISABLED=OFF -DDAS_LLVM_DISABLED=OFF -DDAS_AUDIO_DISABLE | ast-verify changed `.das` | preflight's `ast-verify` gate - ` -dry-run --ast-verify-batch ` per changed `.das` plus the `tests/linq/test_linq_fold.das` qmacro canary, parallel, 300 s per-file timeout, skipping `cant_`/`failed_`/`invalid_` and `utils/internal/ast-fuzz/selftest/`. An `AST verify` line, crash or timeout fails; a compile error belongs to whoever owns the file; a file inside the verifier's own require closure (`daslib/ast*.das`, `daslib/rtti.das`, `daslib/strings_boost.das` - `error[20510]` under the force-include) is reported *not verifiable*, never clean | mirrors the workflow's "Run ast-verify on changed .das files". Batch mode is the ruled gate form (`skills/das_macros.md`); with no pre-infer walk, a tree a macro breaks mid-inference surfaces as a compiler crash instead of a located report - hence crash = red, and plain `--ast-verify` on that file locates it. Each item is a whole-engine compile (2-3x a plain one). Width is physical cores halved; `-j` only lowers it | | REVIEW.das gates | ` utils/internal/review-md/all.das` | every `REVIEW.das` in the tree, fail-fix; also run per-diff in the make_pr step-0a walk | | dastest own suite | ` dastest/dastest.das -- --failures-only --test dastest/tests` | framework suite + `review_gate` library tests; whole-directory, so a new file needs no CI row | -| daslang_static sweep | `cmake --build build --config Release --target daslang_static`, then `bin/Release/daslang_static.exe dastest/dastest.das -- --color --failures-only --test tests` | catches static-registration / no-dynamic-modules divergence | +| daslang_static sweep | `cmake --build build --config Release --target daslang_static`, then `bin/Release/daslang_static.exe dastest/dastest.das -- --color --failures-only --test tests` | the `modules` role; catches static-registration / no-dynamic-modules divergence, which no preflight gate mirrors | +| CI matrix test | preflight's `ci-matrix` gate (fast tier, reach `.github/` and `ci/`); manual: `python3 ci/test_ci_matrix.py` | the per-event cells of `ci/ci_matrix.py` and the `matrix.role != ''` spelling of every `extended_checks.yml` step condition | | Ser/deser sweep | ` dastest/dastest.das -- --test tests --ser serialized.bin` then `... --deser serialized.bin` | after touching AST serialization (`ast_serializer.cpp`, flag-bit additions) | | AST verify tree sweep - **not a PR gate** (the per-PR arm is the row above) | `find tests -name '*.das' ! -name 'cant_*' ! -name 'failed_*' ! -name 'invalid_*' -print0 \| xargs -0 -P8 -n1 timeout 120 --ast-verify-batch -compile-only` - an `AST verify` line is a failure; compile errors are expected (many tests assert one). This one-liner attributes neither a crash (`CRASH:` banner) nor a timeout (rc 124) to its file - for those copy the step's `/tmp/ast_verify_one.sh` helper out of the workflow | runs on `extended_checks.yml`'s 04:00 cron: one daslang process per test file, each re-parsing daslib. Force it early with `gh workflow run extended_checks.yml`. Run locally after touching macro or AST-building code - `skills/das_macros.md` | | Authored-doc code blocks - **not a PR gate** | ` utils/internal/doc-verify/main.das` (exit 0 = every authored RST page's das blocks compile; report at `build/doc_verify/report.json`) | nightly cron + `workflow_dispatch`, posix cells only: ~35 min, one daslang spawn per page. Run locally after editing `doc/source/reference/**` or `doc/source/stdlib/handmade/**`, or after daslib/module API changes docs quote - `skills/internal/doc_sweep.md` | -| MCP tools test | ` dastest/dastest.das -- --color --failures-only --test utils/mcp/test_tools.das` | linux-only in CI, runs anywhere; MCP signature changes break it silently - run after editing `utils/mcp/` | +| MCP tools test | `cmake --build build --config Release --target tree_sitter_daslang` (the grammar library plus the `sgconfig.yml` its post-build step stamps - without them ast-grep knows no `daslang` language), then ` dastest/dastest.das -- --color --failures-only --test utils/mcp/test_tools.das` | the `modules` role; MCP signature changes break it silently - run after editing `utils/mcp/` | | dasImgui build | nothing to install - dasImgui is in-tree (`modules/dasImgui`), built like any default-ON module | external ABI canaries (dasImguiImplot, dasImguiNodeEditor + the rest of the daspkg-index) run in `nightly_daspkg_index.yml`; `skills/internal/abi_break_sweep.md` | -| Coverage | ` dastest/dastest.das -- --cov-path coverage.lcov --color --test tests/language --timeout 1800` + `dascov` | | +| Coverage - **nightly** (linux) | ` dastest/dastest.das -- --cov-path coverage.lcov --color --test tests/language --timeout 1800` + `dascov` | | ## doc.yml - the gates diff --git a/src/builtin/ARCHITECTURE.md b/src/builtin/ARCHITECTURE.md index 31eb81c369..29c383e29c 100644 --- a/src/builtin/ARCHITECTURE.md +++ b/src/builtin/ARCHITECTURE.md @@ -57,3 +57,11 @@ share one. `-no-module-cache` disables the cache outright, over an explicit `-mo ` on the same command line as well as over the default, so a spawner can append it as an override; beside `-ser` / `-deser` - the explicit round-trip halves, whose verdict is the point of the run - the host rejects the command line instead of silently disabling them. + +Every variant is its own record and an engine root's record is 200 MB, so the default directory +is capped: after a writeback `ModuleFileCache::finish` lists the directory's `.dascache` files +and removes the oldest by mtime until it fits `DAS_MODULE_CACHE_LIMIT` megabytes (4096 unless +set; `0` disables eviction), never the record just written. `install` touches the record it +reads, so a record in use is the newest and a stale variant the oldest. Only the default +directory is pruned - an explicit `-module-cache ` is the user's - and the limit variable +is the one `DAS*` name the record key skips, since it decides nothing about a compile. diff --git a/src/builtin/module_builtin_ast_serialize.cpp b/src/builtin/module_builtin_ast_serialize.cpp index 1f512d22d7..e5f6a9d522 100644 --- a/src/builtin/module_builtin_ast_serialize.cpp +++ b/src/builtin/module_builtin_ast_serialize.cpp @@ -9,10 +9,20 @@ #include "daScript/ast/ast_visitor.h" #include "daScript/misc/anyhash.h" #include "daScript/misc/sysos.h" +#include "daScript/misc/env_cfg.h" #include #include #include #include +#if !DAS_NO_FILEIO +#ifdef _WIN32 +#include +#include +#else +#include +#include +#endif +#endif #ifdef _WIN32 #include #include @@ -3305,6 +3315,85 @@ namespace das { } #endif +#if !DAS_NO_FILEIO + static const char * MODULE_CACHE_DEFAULT_DIR = ".jitted_scripts/module_cache/"; + + // DAS_MODULE_CACHE_LIMIT, megabytes: 4096 unless set, 0 = no eviction; garbage keeps the default + static uint64_t moduleCacheLimitBytes () { + uint64_t mb = 4096; + if ( const char * env = get_dasenv_module_cache_limit() ) { + if ( *env ) { + char * end = nullptr; + unsigned long long v = strtoull(env, &end, 10); + if ( end && *end == 0 ) mb = uint64_t(v); + } + } + return mb * 1024ull * 1024ull; + } + + struct CacheRecordInfo { + string path; + uint64_t size; + int64_t mtime; + }; + + static void listCacheRecords ( const string & dir, vector & out ) { +#ifdef _WIN32 + struct _finddata_t c_file; + string findPath = dir + "*.dascache"; + intptr_t hFile = _findfirst(findPath.c_str(), &c_file); + if ( hFile != -1L ) { + do { + out.push_back({dir + c_file.name, uint64_t(c_file.size), int64_t(c_file.time_write)}); + } while ( _findnext(hFile, &c_file) == 0 ); + _findclose(hFile); + } +#else + if ( DIR * d = opendir(dir.c_str()) ) { + while ( dirent * e = readdir(d) ) { + string name = e->d_name; + if ( name.size() < 9 || name.compare(name.size() - 9, 9, ".dascache") != 0 ) continue; + string p = dir + name; + struct stat st; + if ( stat(p.c_str(), &st) == 0 ) out.push_back({p, uint64_t(st.st_size), int64_t(st.st_mtime)}); + } + closedir(d); + } +#endif + } + + static void touchFile ( const string & path ) { +#ifdef _WIN32 + (void) _utime(path.c_str(), nullptr); +#else + (void) utime(path.c_str(), nullptr); +#endif + } + + // LRU eviction of the DEFAULT cache directory down to the limit after a writeback; the record + // just written stays, and a record a run read was touched, so it is never the oldest. An + // explicit -module-cache path is the user's directory and is never pruned. + static void evictModuleCache ( const string & justWrote ) { + size_t dirLen = strlen(MODULE_CACHE_DEFAULT_DIR); + if ( justWrote.compare(0, dirLen, MODULE_CACHE_DEFAULT_DIR) != 0 ) return; + uint64_t limit = moduleCacheLimitBytes(); + if ( limit == 0 ) return; + vector recs; + listCacheRecords(MODULE_CACHE_DEFAULT_DIR, recs); + uint64_t total = 0; + for ( auto & r : recs ) total += r.size; + if ( total <= limit ) return; + sort(recs.begin(), recs.end(), [](const CacheRecordInfo & a, const CacheRecordInfo & b) { + return a.mtime != b.mtime ? a.mtime < b.mtime : a.path < b.path; + }); + for ( auto & r : recs ) { + if ( total <= limit ) break; + if ( r.path == justWrote ) continue; + if ( remove(r.path.c_str()) == 0 ) total -= r.size; + } + } +#endif + string ModuleFileCache::defaultPath ( const string & scriptPath, const string & hostBinary, const string & hostOptions ) { string norm = normalizeFileName(scriptPath.c_str()); size_t slash = norm.find_last_of("/\\"); @@ -3324,7 +3413,8 @@ namespace das { #else for ( char ** e = environ; e && *e; ++e ) { #endif - if ( strncmp(*e, "DAS", 3) == 0 ) envs.push_back(*e); + // the cache's own size cap is a policy on the directory, not a compile input + if ( strncmp(*e, "DAS", 3) == 0 && strncmp(*e, "DAS_MODULE_CACHE_LIMIT=", 23) != 0 ) envs.push_back(*e); } sort(envs.begin(), envs.end()); string key = norm + host + "\n" + hostOptions; @@ -3363,6 +3453,7 @@ namespace das { reader = make_unique(&readStorage, false); reader->quietCache = quiet; env.serializer_read = reader.get(); + touchFile(readFrom); // a record in use is the newest for the eviction's LRU } } if ( !writePath.empty() ) { @@ -3430,6 +3521,7 @@ namespace das { } if ( !res.wrote ) remove(tmpPath.c_str()); // never leave a corpse res.saveFailed = !res.wrote; + if ( res.wrote ) evictModuleCache(writePath); } else { res.saveFailed = true; } diff --git a/src/misc/env_cfg.cpp b/src/misc/env_cfg.cpp index e6b6e4362d..ee3660922d 100644 --- a/src/misc/env_cfg.cpp +++ b/src/misc/env_cfg.cpp @@ -38,6 +38,7 @@ namespace das { const char * get_dasenv_jobque_team_eager_exit () { return das_getenv("DAS_JOBQUE_TEAM_EAGER_EXIT"); } const char * get_dasenv_team_prof () { return das_getenv("DAS_TEAM_PROF"); } const char * get_dasenv_trace_module_load () { return das_getenv("DAS_TRACE_MODULE_LOAD"); } + const char * get_dasenv_module_cache_limit () { return das_getenv("DAS_MODULE_CACHE_LIMIT"); } const char * get_columns () { return das_getenv("COLUMNS"); } } diff --git a/tests-cpp/small/test_table_key_hash.cpp b/tests-cpp/small/test_table_key_hash.cpp new file mode 100644 index 0000000000..c6662899a1 --- /dev/null +++ b/tests-cpp/small/test_table_key_hash.cpp @@ -0,0 +1,103 @@ +// KeyHash (runtime_table.h) is the hash AOT's TTable and every table grow use; the interpreter's +// table nodes, the JIT helpers, the JSON scanner and the C API hash a builtin key as itself and a +// handled key as its annotation's workhorse. Each cell pins that KeyHash agrees, per key type - the +// two rails disagreeing is a table that loses its keys at the first grow (ARCHITECTURE.md, +// "Table key hashing"). + +#include + +#include "daScript/daScript.h" +#include "daScript/simulate/runtime_table.h" +#include "daScript/simulate/jit_abi.h" + +// two handled value types the way a module declares them: a 12-byte vector whose workhorse is +// vec4f (the shape of dasUnitTest's Point3 and dasImgui's ImVec2) and a 4-byte id whose workhorse +// is int32 (dasUnitTest's EntityId) +struct NarrowVec { float x, y, z; }; +struct IdHandle { int32_t value; }; + +namespace das { + template <> struct WrapType { enum { value = true }; typedef vec4f type; typedef vec4f rettype; }; + template <> struct cast : cast_fVec {}; + template <> struct WrapType { enum { value = true }; typedef int32_t type; typedef int32_t rettype; }; + template <> struct cast { + static __forceinline IdHandle to ( vec4f x ) { IdHandle id; id.value = v_extract_xi(v_cast_vec4i(x)); return id; } + static __forceinline vec4f from ( IdHandle x ) { return v_cast_vec4f(v_seti_x(x.value)); } + }; +} + +using namespace das; + +namespace { + +template +void check_builtin ( Context & ctx, const T & key ) { + CHECK_EQ(KeyHash()(ctx, key), hash_function(ctx, key)); +} + +template +uint64_t workhorse_hash ( Context & ctx, const T & key ) { + using workhorse = typename WrapType::type; + return hash_function(ctx, cast::to(cast::from(key))); +} + +} + +TEST_CASE("a builtin table key hashes as itself through KeyHash") { + Context ctx; + check_builtin(ctx, true); + check_builtin(ctx, int8_t(-3)); + check_builtin(ctx, uint8_t(250)); + check_builtin(ctx, int16_t(-1234)); + check_builtin(ctx, uint16_t(65000)); + check_builtin(ctx, int32_t(-7)); + check_builtin(ctx, uint32_t(0xdeadbeefu)); + check_builtin(ctx, int64_t(-1) << 40); + check_builtin(ctx, uint64_t(1) << 63); + check_builtin(ctx, 1.5f); + check_builtin(ctx, 3.25); + Bitfield bf; bf.value = 5u; + check_builtin(ctx, bf); + Bitfield8 bf8; bf8.value = 5u; + check_builtin(ctx, bf8); + Bitfield16 bf16; bf16.value = 5u; + check_builtin(ctx, bf16); + Bitfield64 bf64; bf64.value = 5ull; + check_builtin(ctx, bf64); + check_builtin(ctx, int2(1, 2)); + check_builtin(ctx, int3(1, 2, 3)); + check_builtin(ctx, int4(1, 2, 3, 4)); + check_builtin(ctx, uint2(1u, 2u)); + check_builtin(ctx, uint3(1u, 2u, 3u)); + check_builtin(ctx, uint4(1u, 2u, 3u, 4u)); + check_builtin(ctx, float2(1.f, 2.f)); + check_builtin(ctx, float3(1.f, 2.f, 3.f)); + check_builtin(ctx, float4(1.f, 2.f, 3.f, 4.f)); + check_builtin(ctx, range(3, 9)); + check_builtin(ctx, urange(3u, 9u)); + range64 r64; r64.from = 3; r64.to = 9; + check_builtin(ctx, r64); + urange64 ur64; ur64.from = 3; ur64.to = 9; + check_builtin(ctx, ur64); + char * str = (char *) "key"; + check_builtin(ctx, str); + void * ptr = &ctx; + check_builtin(ctx, ptr); +} + +TEST_CASE("a builtin vector or range key does not take the workhorse detour") { + // the detour hashes a vec4f's 16 bytes; the node hashes the key's own 8 or 12 + Context ctx; + CHECK_NE(KeyHash()(ctx, int2(1, 2)), workhorse_hash(ctx, int2(1, 2))); + CHECK_NE(KeyHash()(ctx, float3(1.f, 2.f, 3.f)), workhorse_hash(ctx, float3(1.f, 2.f, 3.f))); + CHECK_NE(KeyHash()(ctx, range(3, 9)), workhorse_hash(ctx, range(3, 9))); +} + +TEST_CASE("a handled table key hashes as its workhorse through KeyHash") { + Context ctx; + NarrowVec nv { 1.f, 2.f, 3.f }; + CHECK_EQ(KeyHash()(ctx, nv), workhorse_hash(ctx, nv)); + CHECK_NE(KeyHash()(ctx, nv), hash_function(ctx, nv)); // not its 12 raw bytes + IdHandle id { 77 }; + CHECK_EQ(KeyHash()(ctx, id), hash_function(ctx, int32_t(77))); +} diff --git a/tests/language/table_vector_keys.das b/tests/language/table_vector_keys.das new file mode 100644 index 0000000000..f919b33fe8 --- /dev/null +++ b/tests/language/table_vector_keys.das @@ -0,0 +1,90 @@ +options gen2 + +require dastest/testing_boost public +require UnitTest + +// Vector-, range- and handled-keyed tables past their first grow (8 slots): every key inserted +// stays findable, erasable and countable. A second hash definition for a key type shows up +// exactly here - the rehash a grow performs re-buckets with it, and the lookups then miss. A +// builtin key hashes as itself, a handled key as its workhorse; Point3 (12 bytes, workhorse +// vec4f) is the handled type whose raw bytes and workhorse differ. + +let N = 40 + +[test] +def test_int2_keys(t : T?) { + var tab : table // nolint:STYLE027 - one insert at a time is the point: the table must grow and rehash + for (i in range(N)) { + tab[int2(i, i * 7)] = "v{i}" + } + t |> equal(N, length(tab)) + for (i in range(N)) { + t |> equal("v{i}", tab?[int2(i, i * 7)] ?? "", "int2 key {i} after the grows") + t |> success(key_exists(tab, int2(i, i * 7)), "key_exists on int2 key {i}") + } + for (h in range(N / 2)) { + let i = h * 2 + t |> success(tab |> erase(int2(i, i * 7)), "erase of int2 key {i}") + } + t |> equal(N / 2, length(tab)) + for (h in range(N / 2)) { + let i = h * 2 + 1 + t |> equal("v{i}", tab?[int2(i, i * 7)] ?? "", "odd int2 key {i} survives the erases") + } +} + +[test] +def test_float3_keys(t : T?) { + var tab : table // nolint:STYLE027 - one insert at a time is the point: the table must grow and rehash + for (i in range(N)) { + tab[float3(float(i), 0.5, -1.0)] = i + } + t |> equal(N, length(tab)) + for (i in range(N)) { + t |> equal(i, tab?[float3(float(i), 0.5, -1.0)] ?? -1, "float3 key {i} after the grows") + } +} + +[test] +def test_uint2_and_range_keys(t : T?) { + var tab2 : table + var tabr : table + for (i in range(N)) { + tab2[uint2(uint(i), 3u)] = i + tabr[range(i, i + 10)] = i + } + t |> equal(N, length(tab2)) + t |> equal(N, length(tabr)) + for (i in range(N)) { + t |> equal(i, tab2?[uint2(uint(i), 3u)] ?? -1, "uint2 key {i} after the grows") + t |> equal(i, tabr?[range(i, i + 10)] ?? -1, "range key {i} after the grows") + } +} + +[test] +def test_handled_keys(t : T?) { + var ids : table + var pts : table + for (i in range(N)) { + ids[EntityId(i + 100)] = i + pts[Point3(float(i), 2.0, 3.0)] = i + } + t |> equal(N, length(ids)) + t |> equal(N, length(pts)) + for (i in range(N)) { + t |> equal(i, ids?[EntityId(i + 100)] ?? -1, "EntityId key {i} after the grows") + t |> equal(i, pts?[Point3(float(i), 2.0, 3.0)] ?? -1, "Point3 key {i} after the grows") + } +} + +[test] +def test_int4_keys(t : T?) { + var tab : table // nolint:STYLE027 - one insert at a time is the point: the table must grow and rehash + for (i in range(N)) { + tab[int4(i, 1, 2, 3)] = i + } + t |> equal(N, length(tab)) + for (i in range(N)) { + t |> equal(i, tab?[int4(i, 1, 2, 3)] ?? -1, "int4 key {i} after the grows") + } +} diff --git a/tests/language/test_cross_tier_table_hash.das b/tests/language/test_cross_tier_table_hash.das index 903d09d35b..5b8d31adc6 100644 --- a/tests/language/test_cross_tier_table_hash.das +++ b/tests/language/test_cross_tier_table_hash.das @@ -3,24 +3,47 @@ require UnitTest require dastest/testing_boost public var g_tab : table +var g_vec : table +var g_pts : table + +let N = 40 // One table reached by both tiers in one program: [no_aot] keeps the write interpreted inside // the AOT binary, while the read is AOT. The index is written directly here rather than through // a daslib generic, so the hashing happens in this function and not in an AOT'd callee. A value // type is hashed and compared by its workhorse in every tier, so the read finds what the write -// put there; hashing the C++ object in AOT instead puts the key in another bucket. +// put there; hashing the C++ object in AOT instead puts the key in another bucket. A builtin +// vector key is the other direction: the interpreter hashes the int2 itself, so an AOT read that +// hashed its vec4f workhorse would miss. Point3 is a handled type narrower than its workhorse, +// and N keys carry every table past its first grow. [no_aot, no_jit] def write_interpreted { g_tab[EntityId(7)] = 42 + for (i in range(N)) { + g_vec[int2(i, i * 7)] = i + g_pts[Point3(float(i), 2.0, 3.0)] = i + } } def read_aot : int { return g_tab ?[EntityId(7)] ?? -1 } +def read_aot_misses : int { + var misses = 0 + for (i in range(N)) { + misses += (g_vec ?[int2(i, i * 7)] ?? -1) != i ? 1 : 0 + misses += (g_pts ?[Point3(float(i), 2.0, 3.0)] ?? -1) != i ? 1 : 0 + } + return misses +} + [test] def test_cross_tier_table_hash(t : T?) { write_interpreted() t |> equal(length(g_tab), 1) t |> equal(read_aot(), 42) + t |> equal(length(g_vec), N) + t |> equal(length(g_pts), N) + t |> equal(read_aot_misses(), 0, "every int2 and Point3 key the interpreter wrote is found from AOT") } diff --git a/tests/module_cache/ARCHITECTURE.md b/tests/module_cache/ARCHITECTURE.md index 56651c0a68..1c070f807d 100644 --- a/tests/module_cache/ARCHITECTURE.md +++ b/tests/module_cache/ARCHITECTURE.md @@ -12,7 +12,8 @@ this document states what the folder is and why its tests take the shape they do count and the cutoff; on an explicit cache it also pins the two stamps a record carries - the compile's policies, whose change reparses every module in place and rewrites the file, and the module's source, stamped by content so a byte-identical rewrite serves and a same-size - edit cuts off. + edit cuts off; and the directory's size cap - children run inside a temp directory, so the + default cache they fill and prune is their own, never the tree's. - `test_macro_dep_invalidate.das` - a compile-time input a macro pinned through `add_module_cache_dependency` is compared by content, not mtime: a byte-identical rewrite serves the record, a changed file re-parses from that module on and says so. diff --git a/tests/module_cache/test_default_cache_path.das b/tests/module_cache/test_default_cache_path.das index 310c48e85e..dedad61313 100644 --- a/tests/module_cache/test_default_cache_path.das +++ b/tests/module_cache/test_default_cache_path.das @@ -388,3 +388,82 @@ def test_default_module_cache(t : T?) { var err : string rmdir_rec(tmp, err) } + +def records_in(dir, stem : string) : int { + var n = 0 + fio::dir(dir) $(name) { + if (name |> starts_with("{stem}-") && name |> ends_with(".dascache")) { + n++ + } + } + return n +} + +def record_size_in(dir, stem : string) : uint64 { + var size = 0ul + fio::dir(dir) $(name) { + if (name |> starts_with("{stem}-") && name |> ends_with(".dascache")) { + size = stat(path_join(dir, name)).size + } + } + return size +} + +//! the default cache directory holds at most DAS_MODULE_CACHE_LIMIT megabytes: past it the oldest +//! records go, a record read this run counts as fresh, the record just written stays, and an +//! explicit -module-cache path is never pruned +[test] +def test_default_cache_evicts_to_the_limit(t : T?) { + let hostArgs <- get_command_line_arguments() + if (find_index(hostArgs, "--use-aot") >= 0) { + t |> skip("an AOT-consuming host never installs the default cache") + return + } + var terr : string + let tmp = create_temp_directory("das_mc_evict", terr) + if (empty(tmp)) { + t |> failure("create_temp_directory: {terr}") + return + } + let win = get_platform_name() == "windows" + //! the children run inside the temp directory, so the default cache they prune is their own + let enter = win ? "cd /d {tmp} && set \"DAS_MODULE_CACHE_LIMIT=1\"&&" : "cd {tmp} && DAS_MODULE_CACHE_LIMIT=1" + let base = "{enter} {get_full_file_name(das_exe())} -dasroot {get_das_root()}" + let cacheDir = path_join(tmp, ".jitted_scripts/module_cache") + for (stem in ["ev_a", "ev_b", "ev_c", "ev_d"]) { + fwrite(path_join(tmp, "{stem}.das"), "options gen2\n[export]\ndef main \{\n print(\"MARK_{stem}\\n\")\n\}\n") + } + var out : string + t |> success(report_child(t, "evict a", run("{base} {path_join(tmp, "ev_a.das")}", out), out, "MARK_ev_a"), "the first root writes its record") + let one = record_size_in(cacheDir, "ev_a") + t |> success(one > 0ul && 2ul * one <= 1048576ul && 3ul * one > 1048576ul, "a hello record ({one} bytes) sizes so that two fit the 1 MB limit and three do not - retune the limit if daslib grew") + sleep(1100u) //! past the mtime's one-second grain, so the records order by age + t |> success(report_child(t, "evict b", run("{base} {path_join(tmp, "ev_b.das")}", out), out, "MARK_ev_b"), "the second root writes its record") + sleep(1100u) + t |> success(report_child(t, "evict c", run("{base} {path_join(tmp, "ev_c.das")}", out), out, "MARK_ev_c"), "the third root writes its record") + t |> equal(records_in(cacheDir, "ev_a"), 0, "the oldest record went when the third pushed the directory past the limit") + t |> equal(records_in(cacheDir, "ev_b"), 1, "the middle record stays") + t |> equal(records_in(cacheDir, "ev_c"), 1, "the record just written stays") + sleep(1100u) + t |> success(report_child(t, "evict b warm", run("{base} {path_join(tmp, "ev_b.das")}", out), out, "MARK_ev_b"), "a warm run reads the second record") + sleep(1100u) + t |> success(report_child(t, "evict d", run("{base} {path_join(tmp, "ev_d.das")}", out), out, "MARK_ev_d"), "a fourth root writes its record") + t |> equal(records_in(cacheDir, "ev_b"), 1, "the record the warm run read counted as fresh and survived") + t |> equal(records_in(cacheDir, "ev_c"), 0, "the record no run touched since was the oldest and went") + t |> equal(records_in(cacheDir, "ev_d"), 1, "the record just written stays") + let explicitDir = path_join(tmp, "explicit") + for (stem in ["ev_a", "ev_b", "ev_c"]) { + let cachePath = path_join(explicitDir, "{stem}.dascache") + let script = path_join(tmp, "{stem}.das") + t |> success(report_child(t, "explicit {stem}", run("{base} -module-cache {cachePath} {script}", out), out, "ser: wrote"), "an explicit cache writes under the limit too") + } + var explicitFiles = 0 + fio::dir(explicitDir) $(name) { + if (name |> ends_with(".dascache")) { + explicitFiles++ + } + } + t |> equal(explicitFiles, 3, "an explicit -module-cache directory is never pruned") + var err : string + rmdir_rec(tmp, err) +} diff --git a/utils/REVIEW.md b/utils/REVIEW.md index 8c1e20a061..01d288f9ea 100644 --- a/utils/REVIEW.md +++ b/utils/REVIEW.md @@ -17,8 +17,8 @@ what identifies one - the fields that decide whether two `.dlim`s are the same i to `modules/dasLLAMA/REVIEW.md` (repo root) too.** A `utils/` diff never opens that checklist on its own. -**Weakening `REVIEW.das` (beside this file) is a defect.** What the gate checks is read from the -script itself, and each check's finding text states its rule. +**Weakening `REVIEW.das` (beside this file) is a defect: dropping a check, narrowing what a check +walks, or rewriting a finding text so it no longer names what failed.** **A diff that drops a tool from `DAS_UTILS_SHIPPED_EXES` (`CMakeLists.txt`, beside this file) while keeping that tool's directory records the decision to stop shipping it in that tool's @@ -31,9 +31,10 @@ list never carried leaves no record. **A test the diff adds or changes that covers a change under `utils/`, whose load-bearing assertions a CI row can run against the change, ships with a CI row that executes those -assertions, wherever the diff puts the test, added in the same change if no row already covers -it.** A row that only compile-checks the test (`dastest --compile-only`) does -not execute them. A test whose assertions no row executes never runs again. +assertions on every pull request, wherever the diff puts the test, added in the same change if +no row already covers it.** A row that only compile-checks the test (`dastest --compile-only`) +does not execute them, and a nightly-only row runs them after the merge. A test whose +assertions no row executes never runs again. **A test the diff adds or changes that covers a change under `utils/`, whose load-bearing assertions no CI row can run for want of hardware or data, ships with a row that compile-checks it - `dastest diff --git a/utils/internal/make-pr/gates.das b/utils/internal/make-pr/gates.das index f688eddaf9..61a52417c2 100644 --- a/utils/internal/make-pr/gates.das +++ b/utils/internal/make-pr/gates.das @@ -21,6 +21,14 @@ def public want_gate(only : string; skip : array; gate : string) : bool return true } +//! the gates the default chain leaves to preflight (review-md, ast-verify: its fast-tier gates, one pass +//! each) or off the critical path (dupes: an advisory report); each still runs by name through --only +let public NAMED_ONLY_GATES <- fixed_array("review-md", "ast-verify", "dupes") + +def public in_default_chain(only : string; gate : string) : bool { + return !empty(only) || NAMED_ONLY_GATES |> find_index(gate) < 0 +} + def public unknown_gate_names(only : string; skip : array) : array { var bad <- [for (g in skip); g; where GATE_NAMES |> find_index(g) < 0] if (!empty(only) && GATE_NAMES |> find_index(only) < 0) { diff --git a/utils/internal/make-pr/main.das b/utils/internal/make-pr/main.das index 4b12880d6f..154d231b9d 100644 --- a/utils/internal/make-pr/main.das +++ b/utils/internal/make-pr/main.das @@ -306,10 +306,10 @@ def main() : int { return 1 } if ((want("sync") && !gate_sync()) || - (want("review-md") && !gate_review_md(daslang)) || + (in_default_chain(g_cfg.only, "review-md") && want("review-md") && !gate_review_md(daslang)) || (want("stamp-reach") && !gate_stamp_reach()) || - (want("dupes") && !gate_dupes(daslang)) || - (want("ast-verify") && !gate_ast_verify(daslang)) || + (in_default_chain(g_cfg.only, "dupes") && want("dupes") && !gate_dupes(daslang)) || + (in_default_chain(g_cfg.only, "ast-verify") && want("ast-verify") && !gate_ast_verify(daslang)) || (want("jit-smoke") && !gate_jit_smoke(daslang))) { return 2 } diff --git a/utils/internal/make-pr/test_gates.das b/utils/internal/make-pr/test_gates.das index 6be9cfec0b..1d112b4f0d 100644 --- a/utils/internal/make-pr/test_gates.das +++ b/utils/internal/make-pr/test_gates.das @@ -15,6 +15,17 @@ def test_want_gate(t : T?) { t |> success(want_gate("", skip, "sync"), "skip-mode: others still wanted") } +[test] +def test_in_default_chain(t : T?) { + for (g in ["sync", "stamp-reach", "jit-smoke"]) { + t |> success(in_default_chain("", g), "{g} is in the default chain") + } + for (g in NAMED_ONLY_GATES) { + t |> success(!in_default_chain("", g), "{g} is not in the default chain") + t |> success(in_default_chain(g, g), "--only {g} runs it") + } +} + [test] def test_unknown_gate_names(t : T?) { let no_skip : array diff --git a/utils/internal/preflight/README.md b/utils/internal/preflight/README.md index d6e10acd0d..59604852dc 100644 --- a/utils/internal/preflight/README.md +++ b/utils/internal/preflight/README.md @@ -1,21 +1,28 @@ # preflight -Run CI's gates locally before pushing. The CI-lane <-> gate mapping and the +Run CI's gates locally before pushing. The CI-lane <-> gate mapping, the tiers and the manual commands this tool automates live in -[skills/internal/preflight.md](https://github.com/GaijinEntertainment/daScript/blob/master/skills/internal/preflight.md). +[skills/internal/preflight.md](https://github.com/GaijinEntertainment/daScript/blob/master/skills/internal/preflight.md); +the budget the tiers serve (a full run fits 20 minutes on the M5 box) is +`plans/ci_preflight_budget.md`. ```bash -# fast tier: format --verify, lint changed .das, ast-verify changed .das (batch mode, -# parallel, 300s/file - each item is a whole-engine compile, so a sweep-shaped diff is -# minutes), clang frontend pass on changed C++ (full src+tests-cpp sweep when a header changed) +# fast tier, serial, a red stops the run: untracked, format --verify, lint changed .das (three +# rails), hash-refs, review-md, md-ascii, ast-verify changed .das (batch mode, parallel, 300s/file), +# clang frontend pass on changed C++, dasgen freshness, ci-das, ci-matrix, compile-sweep (every +# program root under utils/, examples/, tutorials/ and the modules' examples and utils, in parallel) daslang utils/internal/preflight/main.das -# full tier: adds the untracked-files gate (working tree carries none - commit, -# delete, or ignore each), dasgen freshness, CI-only-das compile sweep, the -# six doc gates, ctest -L small, interpreter/JIT/AOT suites, sequence smoke +# --full adds the lanes, every one at once so the wall is the longest: docs (the seven doc.yml +# gates), tests-cpp, tests-interp, tests-jit, tests-aot (builds test_aot first), utils-tests daslang utils/internal/preflight/main.das -- --full +daslang utils/internal/preflight/main.das -- --full --serial # the lanes one after another, for diagnosis -# subset / introspection +# module gates never run from a tier - name them when the module is the work +daslang utils/internal/preflight/main.das -- --only imgui +daslang utils/internal/preflight/main.das -- --only dasllama-model-free + +# subset / introspection: --list-gates prints tier, reach and description daslang utils/internal/preflight/main.das -- --list-gates daslang utils/internal/preflight/main.das -- --only docs,ci-das daslang utils/internal/preflight/main.das -- --skip tests-aot --full @@ -25,9 +32,15 @@ daslang utils/internal/preflight/main.das -- --skip tests-aot --full daslang utils/internal/preflight/main.das -- --only lint --lint-skip-exe-rail ``` +A gate with a reach set (`dasgen`, `docs`, `utils-tests`, `ci-matrix`, the module gates) skips +with the reason when nothing under its paths or the core (`src/`, `include/`, `daslib/`, +`dastest/`, `CMakeLists.txt`, `cmake/`) changed against `--base` (default `origin/master`); +`--only` runs a gate whatever changed. Each verdict line carries its breakdown indented beneath +it, and the run closes with a time-by-gate table. + A complete `--full` run requires a Release host and fails immediately when given `bin/Debug/daslang`. Debug remains available for focused `--only` or -`--skip` diagnosis, but running the entire ~12k-file matrix under it is never +`--skip` diagnosis, but running the entire ~13k-file matrix under it is never an acceptable pre-push substitute. On Windows, if the Release runtime DLL is locked by a `utils/mcp/main.das` process, stop that worktree's MCP host and retry the Release build; its watcher restarts it. Do not fall back to Debug. @@ -54,17 +67,18 @@ own smoke scripts under pwsh/bash. The C++ pass uses `clang-cl /Zs` on Windows (preferring the VS-bundled clang - the same binary CI's ClangCL toolset uses) and `clang -fsyntax-only` elsewhere; both are frontend-only (parse + semantic analysis + template instantiation, no codegen), which keeps -even the full ~160-TU header-change sweep at ~15-30 s. A gate whose host tool +even the full ~200-TU header-change sweep at ~15-30 s. A gate whose host tool or module is missing reports `SKIP` with an install/rebuild hint instead of passing silently. Exit code is non-zero when any gate fails. -The full interpreter and AOT sweeps pass `--max-file-time 30` to `dastest`. -The parallel cold JIT sweep defaults to 60 seconds because healthy files can -cross 30 seconds under worker contention. Any completed test file above its -ceiling fails preflight even when its assertions pass; the suite-wide timeout -remains the separate deadlock guard. +The full interpreter and AOT sweeps pass `--max-file-time` to `dastest`. The +parallel cold JIT sweep defaults to 60 seconds because healthy files can cross +30 seconds under worker contention. Any completed test file above its ceiling +fails preflight even when its assertions pass; the suite-wide timeout remains +the separate deadlock guard. `ci_only_das.txt` lists the in-repo das surface that no default local build compiles (dasOpenGL today); see the header comment there before adding entries - surfaces that pull external daspkg packages belong to the -`sequence` gate, not the compile sweep. +`sequence` gate, not the compile sweep. The compile sweep's own exclusions, +each with its reason, are `SWEEP_EXCLUDED` in `config.das`. diff --git a/utils/internal/preflight/config.das b/utils/internal/preflight/config.das index db29566a17..170991d6ed 100644 --- a/utils/internal/preflight/config.das +++ b/utils/internal/preflight/config.das @@ -31,6 +31,155 @@ def public has_path_under(paths : array; prefix : string) : bool { return false } +def public has_path_under_any(paths : array; prefixes : array) : bool { + for (prefix in prefixes) { + if (has_path_under(paths, prefix)) { + return true + } + } + return false +} + +//! the paths whose change reaches every gate that names a reach set +let public CORE_REACH <- ["src/", "include/", "daslib/", "dastest/", "CMakeLists.txt", "cmake/"] + +//! The pure half of a gate's reach test: a gate with an empty reach set runs whatever changed; one +//! with a reach set runs when a changed path sits under one of its prefixes or under the core. +def public reach_hit(changed : array; reach : array) : bool { + return true if (empty(reach)) + return has_path_under_any(changed, reach) || has_path_under_any(changed, CORE_REACH) +} + +//! The pure half of the compile sweep's root filter: a `.das` under the swept trees that is not a test, +//! not generated, and not a module tree reached through a junction inside an example. +def public is_sweep_root_path(path : string) : bool { + let p = path |> replace("\\", "/") + let name = base_name(p) + // "/modules/" inside the path is a module tree nested in an example - a junction the module's own gates cover + return ((p |> ends_with(".das")) + && !(name |> starts_with("test_")) && !(name |> starts_with("_")) + && find(p, "/tests/") < 0 && find(p, "/_aot_generated/") < 0 + && find(p, "/modules/") < 0) +} + +//! the tier rule: the fast tier runs on every run, the lane tier under --full, the module tier never +def public tier_runs(tier : string; full : bool) : bool { + return tier == "fast" || (full && tier == "lane") +} + +//! trees the compile sweep leaves alone, each with why: a native module a CMake option builds, a resolution the +//! plain compile cannot do, a platform the box is not, fixtures that are broken on purpose +let public SWEEP_EXCLUDED <- { + "examples/crash/" => "the native `crash` module, built only under its CMake option", + "examples/daStrudel/sfx_lab/" => "a project-root example: its modules resolve through -project_root", + "examples/daspkg/" => "daspkg package examples: roots resolve through daspkg install", + "examples/fatman/" => "WASM-only programs", + "examples/games/sequence/" => "GLFW-gated: the sequence gate compiles and runs it", + "modules/dasClangBind/examples/" => "needs libclang (opt-in module)", + "modules/dasSMT/examples/" => "needs z3 (opt-in module)", + "utils/internal/ast-fuzz/selftest/" => "deliberately broken fixtures test_ast_fuzz.das owns" +} + +def public sweep_excluded(path : string) : bool { + for (prefix in keys(SWEEP_EXCLUDED)) { + return true if (path |> starts_with(prefix)) + } + return false +} + +//! a program root is a file with an exported main; a module or a fixture is not +def public is_sweep_root_text(text : string) : bool { + return find(text, "[export]") >= 0 && find(text, "def main") >= 0 +} + +//! the require whose roots pay the engine compile (~24 s each on the M5 box) - they run serially +//! through one shared module cache instead of contending in the pool +let public SWEEP_HEAVY_REQUIRE = "require dasllama/" + +def public is_heavy_sweep_root(text : string) : bool { + return find(text, SWEEP_HEAVY_REQUIRE) >= 0 +} + +//! The files dastest's closing FAILURES block names, comma-joined, at most five. +def public failed_files_block(out : string) : string { + var files : array + var inside = false + for (raw in split(out, "\n")) { + let ln = strip(raw) + if (ln == "FAILURES:") { + inside = true + continue + } + continue if (!inside) + break if (empty(ln)) + let dash = find(ln, " — ") + files |> push(dash >= 0 ? slice(ln, 0, dash) : ln) + } + if (length(files) > 5) { + let more = length(files) - 5 + files |> resize(5) + files |> push("and {more} more") + } + return empty(files) ? "the suite (no FAILURES block in the output)" : join(files, ", ") +} + +struct public GateReport { + tag : string + name : string + seconds : double + detail : string + breakdown : string +} + +//! A lane child's gate lines, `[TAG] name (12.3s) — detail`, each with the indented breakdown under it, +//! in print order. A child that died before its report yields none; the docs lane yields several. +//! A captured child writes through to_log, so every line may carry a `[I] ` / `[W] ` / `[E] ` level prefix. +def public parse_gate_reports(out : string) : array { + var reports : array + var breakdown : array + var open_report = false + for (raw in split(out, "\n")) { + var ln = raw |> replace("\r", "") + if ((ln |> starts_with("[I] ")) || (ln |> starts_with("[W] ")) || (ln |> starts_with("[E] "))) { + ln = slice(ln, 4) + } + if (open_report && ln |> starts_with(" ") && !empty(strip(ln))) { + breakdown |> push(strip(ln)) + continue + } + if (open_report) { + reports[length(reports) - 1].breakdown = join(breakdown, "\n") + breakdown |> clear() + open_report = false + } + continue if (!(ln |> starts_with("["))) + let close = find(ln, "]") + continue if (close < 0) + let tag = strip(slice(ln, 1, close)) + continue if (tag != "PASS" && tag != "FAIL" && tag != "SKIP") + let rest = slice(ln, close + 1) + var head = strip(rest) + var seconds = 0.0lf + var detail = "" + let dash = find(head, " — ") + if (dash >= 0) { + detail = strip(slice(head, dash + length(" — "))) + head = strip(slice(head, 0, dash)) + } + let open = find(head, " (") + if (open >= 0 && head |> ends_with("s)")) { + seconds = double(to_float(slice(head, open + 2, length(head) - 2))) + head = strip(slice(head, 0, open)) + } + reports |> push(GateReport(tag = tag, name = head, seconds = seconds, detail = detail)) + open_report = true + } + if (open_report) { + reports[length(reports) - 1].breakdown = join(breakdown, "\n") + } + return <- reports +} + def public valid_jit_max_file_time_override(value : float) : bool { return value == -1.0 || value >= 0.0 } diff --git a/utils/internal/preflight/main.das b/utils/internal/preflight/main.das index 55ff321428..e553bf0b9b 100644 --- a/utils/internal/preflight/main.das +++ b/utils/internal/preflight/main.das @@ -33,9 +33,12 @@ require compile_db [CommandLineArgs] struct Config { - @clarg_doc = "Run the full tier (adds untracked, dasgen, ci-das, docs, tests-cpp, interp/JIT/AOT suites, sequence smoke)" + @clarg_doc = "Run the fast tier and then every lane at once (docs, tests-cpp, interp/JIT/AOT suites, utils tests); module gates need --only" full : bool + @clarg_doc = "Lanes run serially instead of concurrently (diagnosis; the wall becomes the sum)" + serial : bool + @clarg_doc = "Diff base for changed-file detection (default: origin/master)" base : string = "origin/master" @@ -112,6 +115,7 @@ struct PreflightCtx { base : string changed_das : array changed_cpp : array + changed_all : array // every changed path vs base, any extension, plus untracked files - the reach tests read this changed_hdr : int jobs : int jit_jobs : int @@ -366,6 +370,36 @@ def collect_changed_files(base : string; var das_files, cpp_files : array { + var seen : table + var out : array + let probe = run_argv(["git", "rev-parse", "--verify", "--quiet", base]) + var ranges : array + if (probe.rc == 0) { + ranges |> push("{base}...HEAD") + } + ranges |> push("HEAD") + for (r in ranges) { + let d = run_argv(["git", "diff", "--name-only", r]) + continue if (d.rc != 0) + for (f in non_empty_lines(d.out)) { + continue if (key_exists(seen, f)) + seen |> insert(f) + out |> push(f) + } + } + let u = run_argv(["git", "ls-files", "--others", "--exclude-standard"]) + if (u.rc == 0) { + for (f in non_empty_lines(u.out)) { + continue if (key_exists(seen, f)) + seen |> insert(f) + out |> push(f) + } + } + return <- out +} + // ===== gate runners ===== // PR-time hygiene: leftover session goo must not ride into or linger past a PR. @@ -1129,6 +1163,179 @@ def gate_ci_das(ctx : PreflightCtx) : GateResult { seconds = seconds_since(t0), detail = detail) } +//! the CI matrix test: ci/ci_matrix.py's per-event cells and the extended_checks role conditions +def gate_ci_matrix() : GateResult { + let t0 = ref_time_ticks() + if (!tool_available("python3", "--version")) { + return GateResult(name = "ci-matrix", status = GateStatus.Skip, seconds = seconds_since(t0), detail = "python3 not found on PATH") + } + let r = run_argv(["python3", "ci/test_ci_matrix.py"], 120.0) + return GateResult(name = "ci-matrix", status = r.rc == 0 ? GateStatus.Pass : GateStatus.Fail, seconds = seconds_since(t0), + detail = r.rc == 0 ? "" : "ci/test_ci_matrix.py red - a workflow matrix cell or an extended_checks role condition (must be matrix.role != '')", + output = r.out) +} + +//! the trees whose program roots the compile sweep compiles +let SWEEP_TREES <- ["utils", "examples", "tutorials"] + +def private collect_das_files(dir : string; var out : array&) { + fio::dir(dir) $(name) { + return if (name == "." || name == "..") + let p = "{dir}/{name}" + var st : FStat + if (stat(p, st) && st.is_dir) { + collect_das_files(p, out) + } elif (name |> ends_with(".das")) { + out |> push(p) + } + } +} + +//! every program root under the swept trees - utils/, examples/, tutorials/ and the modules' examples, +//! tutorials and utils folders - split into the light pool and the engine-heavy serial set +def collect_sweep_roots() : tuple; heavy : array> { + var files : array + for (tree in SWEEP_TREES) { + collect_das_files(tree, files) + } + fio::dir("modules") $(mod) { + return if (mod == "." || mod == "..") + for (sub in ["examples", "tutorials", "utils"]) { + let p = "modules/{mod}/{sub}" + var st : FStat + if (stat(p, st) && st.is_dir) { + collect_das_files(p, files) + } + } + } + var light : array + var heavy : array + for (f in files) { + continue if (sweep_excluded(f) || !is_sweep_root_path(f)) + let text = fread(f) + continue if (!is_sweep_root_text(text)) + if (is_heavy_sweep_root(text)) { + heavy |> push(f) + } else { + light |> push(f) + } + } + sort(light) + sort(heavy) + return <- (light <- light, heavy <- heavy) +} + +struct SweepItem { + index : int + path : string +} + +struct SweepResult { + index : int + rc : int + out : string +} + +//! one `daslang -compile-only` per root, `workers` in flight - the pooled shape of run_verify_workers +def private run_sweep_workers(daslang : string; files : array; workers : int; timeout : float) : array { + let n = length(files) + var results : array + results |> resize(n) + return <- results if (n == 0) + let nw = clamp(workers, 1, n) + with_channel(n) $(inCh) { + with_channel(n) $(outCh) { + with_job_status(nw) $(done) { + for (_t in range(nw)) { + new_thread <| @ { + for_each_clone(inCh) $(item : SweepItem#) { + var out : string + let path = string(item.path) + let argv <- [daslang, "-compile-only", "-dasroot", ".", path] + let rc = unsafe(popen_argv(argv, timeout, $(f) { + if (f != null) { + out := fread(f) + } + })) + outCh |> push_clone(SweepResult(index = item.index, rc = rc, out = out)) + outCh |> notify() + } + inCh |> release() + outCh |> release() + done |> notify_and_release() + } + } + for (i in range(n)) { + inCh |> push_clone(SweepItem(index = i, path = files[i])) + inCh |> notify() + } + for (r in each_clone(outCh, type)) { + if (r.index >= 0 && r.index < n) { + results[r.index].index = r.index + results[r.index].rc = r.rc + results[r.index].out := r.out + } + } + done |> join() + } + } + } + return <- results +} + +//! the per-PR form of CI's examples and tutorial runs: does every program root still compile +def gate_compile_sweep(ctx : PreflightCtx) : GateResult { + let t0 = ref_time_ticks() + let roots <- collect_sweep_roots() + let total = length(roots.light) + length(roots.heavy) + if (total == 0) { + return GateResult(name = "compile-sweep", status = GateStatus.Skip, seconds = seconds_since(t0), detail = "no program roots found under {join(SWEEP_TREES, ", ")}") + } + //! cores, not get_total_hw_threads(): the latter is the jobque's own worker count (5 on an 18-core M5) + let workers = compute_worker_count(length(roots.light), ctx.jobs > 0 ? ctx.jobs : get_total_hw_cores()) + let t_pool = ref_time_ticks() + let light <- run_sweep_workers(ctx.daslang, roots.light, workers, 300.0) + let pool_s = seconds_since(t_pool) + // the engine roots share one module graph: one explicit cache, written by the first, served to the rest + let cache = unique_temp_path("preflight_sweep", ".dascache") + let t_serial = ref_time_ticks() + var red = 0 + let output = build_string() $(w) { + for (root, r in roots.light, light) { + continue if (r.rc == 0) + red ++ + w |> write("=== {root} (exit {r.rc}) ===\n{r.out}\n") + } + for (root in roots.heavy) { + let r = run_argv([ctx.daslang, "-compile-only", "-dasroot", ".", "-module-cache", cache, root], 600.0) + continue if (r.rc == 0) + red ++ + w |> write("=== {root} (exit {r.rc}) ===\n{r.out}\n") + } + } + let serial_s = seconds_since(t_serial) + remove(cache) + let shape = "{length(roots.light)} in a {workers}-wide pool ({fmt(":.0f", float(pool_s))}s), {length(roots.heavy)} engine roots serial through one module cache ({fmt(":.0f", float(serial_s))}s); {length(SWEEP_EXCLUDED)} trees excluded" + if (red > 0) { + return GateResult(name = "compile-sweep", status = GateStatus.Fail, seconds = seconds_since(t0), + detail = "{red} of {total} program root(s) fail to compile - {shape}", output = output) + } + return GateResult(name = "compile-sweep", status = GateStatus.Pass, seconds = seconds_since(t0), + detail = "{total} program root(s) compile - {shape}") +} + +//! CI's "Run utils tests" step: every utils/*/tests suite through the cmake target +def gate_utils_tests(ctx : PreflightCtx) : GateResult { + let t0 = ref_time_ticks() + if (empty(ctx.build_dir)) { + return GateResult(name = "utils-tests", status = GateStatus.Skip, seconds = seconds_since(t0), + detail = "no configured build dir — cmake -B build first") + } + return run_test_gate("utils-tests", + ["cmake", "--build", ctx.build_dir, "--config", ctx.config, "--target", "run_utils_tests"], + "utils test suites failed (cmake --target run_utils_tests)") +} + // ===== test suites ===== def slowest_files_block(out : string) : string { @@ -1272,9 +1479,17 @@ def gate_tests_aot(ctx : PreflightCtx) : GateResult { detail = "test_aot binary not found next to daslang ({test_aot})") } // 2x CI's per-file cap - same chained-load reasoning as gate_tests_interp - return run_test_gate("tests-aot", + var g <- run_test_gate("tests-aot", [test_aot, "-use-aot", "dastest/dastest.das", "--", "--use-aot", "--failures-only", "--timing-outliers", "10", "--max-file-time", "60", "--timeout", "1800", "--test", "tests"], - "AOT suite failures (error[50101] → skills/internal/aot_hash_desync_debugging.md)", t0) + "AOT suite failures", t0) + if (g.status == GateStatus.Fail) { + if (find(g.output, "error[50101]") >= 0) { + g.detail = "AOT link failed: error[50101] → skills/internal/aot_hash_desync_debugging.md" + } else { + g.detail = "AOT-only failures in {failed_files_block(g.output)} — green under -jit/interp means the AOT emitter (daslib/aot_cpp.das) or aot.h" + } + } + return <- g } def gate_sequence(ctx : PreflightCtx) : GateResult { @@ -1376,36 +1591,46 @@ def gate_dasllama(ctx : PreflightCtx; suite : string) : GateResult { // ===== orchestration ===== +//! tiers: `fast` runs serially first on every run and a red stops the run; `lane` runs only under --full, +//! every lane at once, so the wall is the longest lane; `module` never runs from a tier - `--only ` +//! runs it when the module is the work. `reach`: the path prefixes whose change reaches the gate (the core +//! reaches everything; empty = the gate runs whatever changed) struct GateInfo { name : string - full_only : bool + tier : string doc : string + reach : array } def gate_table() : array { return <- [ - GateInfo(name = "untracked", full_only = true, doc = "no untracked files at PR time — commit, delete, or ignore each"), - GateInfo(name = "format", full_only = false, doc = "formatter --verify on tracked .das (mirrors CI)"), - GateInfo(name = "lint", full_only = false, doc = "lint changed .das on three rails (host, linux-mirror, -exe), zero warnings; --lint-skip-exe-rail drops the exe rail"), - GateInfo(name = "hash-refs", full_only = false, doc = "no bare #N in branch commit messages that GitHub would mislink - ledger cites spell out or backtick"), - GateInfo(name = "review-md", full_only = false, doc = "REVIEW.das gates of every folder the diff touches (utils/internal/review-md; mirrors CI's extended_checks step)"), - GateInfo(name = "md-ascii", full_only = false, doc = "ci/fix_md_ascii.py --check when the diff touches .md (mirrors CI's Markdown ASCII gate in extended_checks)"), - GateInfo(name = "ast-verify", full_only = false, doc = "daslang -dry-run --ast-verify-batch on changed .das (parallel, 300s/file; mirrors CI) - an AST verify line, a crash, or a timeout fails"), - GateInfo(name = "cpp-syntax", full_only = false, doc = "clang frontend pass on changed C++; header change → full src+tests-cpp sweep"), - GateInfo(name = "dasgen", full_only = true, doc = "gen_bind.das freshness vs include/daScript/builtin/"), - GateInfo(name = "ci-das", full_only = true, doc = "compile-only sweep of CI-only das surface (ci_only_das.txt)"), - GateInfo(name = "docs", full_only = true, doc = "the seven doc.yml gates (das2rst, imgui2rst, vulkan2rst, stubs, uncategorized, docs/untracked, sphinx html)"), - GateInfo(name = "tests-cpp", full_only = true, doc = "ctest -L small"), - GateInfo(name = "tests-interp", full_only = true, doc = "full interpreter suite"), - GateInfo(name = "tests-jit", full_only = true, doc = "full JIT suite (skips when dasLLVM absent)"), - GateInfo(name = "tests-aot", full_only = true, doc = "build test_aot + full AOT suite"), - GateInfo(name = "sequence", full_only = true, doc = "sequence game release smoke (only pre-merge compile of GLFW-gated das)"), - GateInfo(name = "imgui", full_only = true, doc = "dasImgui playwright suite (nightly-only CI — this local mirror is the only pre-merge gate for it)"), - GateInfo(name = "dasllama-model-free", full_only = true, doc = "dasLLAMA model-free suite via tests/run.das (skips unless the diff touches modules/dasLLAMA/; no CI lane)"), - GateInfo(name = "dasllama-stocked", full_only = true, doc = "dasLLAMA stocked suite - the model-gated files, a run of skips without models (skips unless the diff touches modules/dasLLAMA/; no CI lane)") + GateInfo(name = "untracked", tier = "fast", doc = "no untracked files at PR time — commit, delete, or ignore each"), + GateInfo(name = "format", tier = "fast", doc = "formatter --verify on tracked .das (mirrors CI)"), + GateInfo(name = "lint", tier = "fast", doc = "lint changed .das on three rails (host, linux-mirror, -exe), zero warnings; --lint-skip-exe-rail drops the exe rail"), + GateInfo(name = "hash-refs", tier = "fast", doc = "no bare #N in branch commit messages that GitHub would mislink - ledger cites spell out or backtick"), + GateInfo(name = "review-md", tier = "fast", doc = "REVIEW.das gates of every folder the diff touches (utils/internal/review-md; CI's extended_checks step runs every gate)"), + GateInfo(name = "md-ascii", tier = "fast", doc = "ci/fix_md_ascii.py --check when the diff touches .md (mirrors CI's Markdown ASCII gate in extended_checks)"), + GateInfo(name = "ast-verify", tier = "fast", doc = "daslang -dry-run --ast-verify-batch on changed .das (parallel, 300s/file; mirrors CI) - an AST verify line, a crash, or a timeout fails"), + GateInfo(name = "cpp-syntax", tier = "fast", doc = "clang frontend pass on changed C++; header change → full src+tests-cpp sweep"), + GateInfo(name = "dasgen", tier = "fast", doc = "gen_bind.das freshness vs include/daScript/builtin/", reach <- ["src/builtin/", "include/daScript/builtin/", "utils/internal/dasgen/"]), + GateInfo(name = "ci-das", tier = "fast", doc = "compile-only sweep of CI-only das surface (ci_only_das.txt)"), + GateInfo(name = "ci-matrix", tier = "fast", doc = "python3 ci/test_ci_matrix.py - the CI job matrices per event and the extended_checks role conditions (mirrors CI's core-role step)", reach <- [".github/", "ci/"]), + GateInfo(name = "compile-sweep", tier = "fast", doc = "compile-only of every program root under utils/, examples/, tutorials/ and the modules' examples and utils, in parallel (the per-PR form of CI's examples and tutorial runs)"), + GateInfo(name = "docs", tier = "lane", doc = "the seven doc.yml gates (das2rst, imgui2rst, vulkan2rst, stubs, uncategorized, docs/untracked, sphinx html)", reach <- ["doc/", "daslib/", "src/builtin/", "modules/dasImgui/", "modules/dasVulkan/", "modules/dasLLAMA/dasllama/"]), + GateInfo(name = "tests-cpp", tier = "lane", doc = "ctest -L small"), + GateInfo(name = "tests-interp", tier = "lane", doc = "full interpreter suite"), + GateInfo(name = "tests-jit", tier = "lane", doc = "full JIT suite (skips when dasLLVM absent)"), + GateInfo(name = "tests-aot", tier = "lane", doc = "build test_aot + full AOT suite (per-PR CI compiles only the subset; the sweep is nightly's)"), + GateInfo(name = "utils-tests", tier = "lane", doc = "run_utils_tests - every utils/*/tests suite (mirrors CI's extended_checks step)", reach <- ["utils/"]), + GateInfo(name = "sequence", tier = "module", doc = "sequence game release smoke; nightly CI - run it when working on the game", reach <- ["examples/games/sequence/"]), + GateInfo(name = "imgui", tier = "module", doc = "dasImgui playwright suite; nightly CI - run it when working on dasImgui", reach <- ["modules/dasImgui/", "modules/dasGlfw/", "modules/dasClipboard/"]), + GateInfo(name = "dasllama-model-free", tier = "module", doc = "dasLLAMA model-free suite via tests/run.das - the module's every-change gate (run.das -- --changed while working)", reach <- ["modules/dasLLAMA/"]), + GateInfo(name = "dasllama-stocked", tier = "module", doc = "dasLLAMA stocked suite - the model-gated files, the module's per-PR run on a box with models", reach <- ["modules/dasLLAMA/"]) ] } +//! --only and --skip name gates outright; otherwise the fast tier always runs, the lane tier under --full, +//! and the module tier never def want_gate(cfg : Config; info : GateInfo) : bool { let only_list <- split_csv(cfg.only) let skip_list <- split_csv(cfg.skip) @@ -1418,7 +1643,58 @@ def want_gate(cfg : Config; info : GateInfo) : bool { } return false } - return cfg.full || !info.full_only + return tier_runs(info.tier, cfg.full) +} + +//! one lane = one child preflight running `--only ` with the parent's settings; every lane at once +//! (one thread per child), so the wall is the longest lane. The child's own verdict lines are the result. +def private run_lanes(ctx : PreflightCtx; cfg : Config; names : array) : array { + var argvs : array> + argvs |> reserve(length(names)) + for (name in names) { + var av <- [ctx.daslang, "utils/internal/preflight/main.das", "--", "--only", name, "--base", cfg.base, + "--daslang", ctx.daslang, "--build-dir", cfg.build_dir] + if (cfg.jobs > 0) { + av |> push("-j") + av |> push("{cfg.jobs}") + } + if (cfg.jit_jobs >= 0) { + av |> push("--jit-jobs") + av |> push("{cfg.jit_jobs}") + } + if (cfg.jit_max_file_time >= 0.0) { + av |> push("--jit-max-file-time") + av |> push("{cfg.jit_max_file_time}") + } + if (!empty(cfg.config_file)) { + av |> push("--config-file") + av |> push(cfg.config_file) + } + if (cfg.verbose) { + av |> push("-v") + } + argvs |> emplace(av) + } + let children <- run_chunk_workers(argvs) + var results : array + results |> reserve(length(names) + 8) // the docs lane reports its seven gates + for (name, child in names, children) { + let reports <- parse_gate_reports(child.stdout) + if (empty(reports)) { + results |> emplace(GateResult(name = name, status = GateStatus.Fail, + detail = "lane child exited {child.exit_code} without a verdict", output = child.stdout)) + continue + } + for (rep in reports) { + var r = GateResult(name = rep.name, seconds = rep.seconds, detail = rep.detail, breakdown = rep.breakdown) + r.status = rep.tag == "PASS" ? GateStatus.Pass : (rep.tag == "SKIP" ? GateStatus.Skip : GateStatus.Fail) + if (r.status == GateStatus.Fail || cfg.verbose) { + r.output = child.stdout + } + results |> emplace(r) + } + } + return <- results } def status_tag(s : GateStatus) : string { @@ -1465,7 +1741,8 @@ def main() : int { // nolint:STYLE037,STYLE038 — the gate loop + CLI surface; } if (cfg.list_gates) { for (info in gate_table()) { - to_log(LOG_INFO, "{info.name}\t{info.full_only ? "full" : "fast"}\t{info.doc}\n") + let reach = empty(info.reach) ? "any change" : "reach: {join(info.reach, " ")} (+core)" + to_log(LOG_INFO, "{info.name}\t{info.tier}\t{reach}\t{info.doc}\n") } return 0 } @@ -1528,6 +1805,7 @@ def main() : int { // nolint:STYLE037,STYLE038 — the gate loop + CLI surface; ctx.clang = clang_info.exe ctx.clang_is_cl = clang_info.is_cl collect_changed_files(cfg.base, ctx.changed_das, ctx.changed_cpp, ctx.changed_hdr) + ctx.changed_all <- collect_changed_paths(cfg.base) let tier = cfg.full ? "full" : "fast" let n_das = length(ctx.changed_das) @@ -1538,8 +1816,21 @@ def main() : int { // nolint:STYLE037,STYLE038 — the gate loop + CLI surface; var results : array let t_all = ref_time_ticks() + var lanes : array + let by_only = !empty(cfg.only) // a gate named outright runs whatever changed - "working on imgui: run imgui" for (info in gate_table()) { continue if (!want_gate(cfg, info)) + if (!by_only && !reach_hit(ctx.changed_all, info.reach)) { + var r = GateResult(name = info.name, status = GateStatus.Skip, + detail = "nothing under {join(info.reach, ", ")} or the core changed vs {cfg.base}") + report_gate(r, cfg.verbose) + results |> emplace(r) + continue + } + if (info.tier == "lane" && !by_only && !cfg.serial) { + lanes |> push(info.name) // deferred: every lane runs at once after the fast tier + continue + } to_log(LOG_INFO, "[RUN ] {info.name}\n") if (info.name == "docs") { var doc_results : array @@ -1570,6 +1861,8 @@ def main() : int { // nolint:STYLE037,STYLE038 — the gate loop + CLI surface; r <- gate_dasgen(ctx) } elif (info.name == "ci-das") { r <- gate_ci_das(ctx) + } elif (info.name == "ci-matrix") { + r <- gate_ci_matrix() } elif (info.name == "tests-cpp") { r <- gate_tests_cpp(ctx) } elif (info.name == "tests-interp") { @@ -1586,6 +1879,10 @@ def main() : int { // nolint:STYLE037,STYLE038 — the gate loop + CLI surface; r <- gate_dasllama(ctx, "model-free") } elif (info.name == "dasllama-stocked") { r <- gate_dasllama(ctx, "stocked") + } elif (info.name == "compile-sweep") { + r <- gate_compile_sweep(ctx) + } elif (info.name == "utils-tests") { + r <- gate_utils_tests(ctx) } report_gate(r, cfg.verbose) results |> emplace(r) @@ -1599,6 +1896,27 @@ def main() : int { // nolint:STYLE037,STYLE038 — the gate loop + CLI surface; } break if (cfg.fail_fast && any_fail) } + if (!empty(lanes)) { + var fast_red = false + for (r in results) { + fast_red ||= r.status == GateStatus.Fail + } + if (fast_red) { + results |> reserve(length(results) + length(lanes)) + for (name in lanes) { // the expensive lanes wait for a green fast tier + var r = GateResult(name = name, status = GateStatus.Skip, detail = "not run: the fast tier is red") + report_gate(r, cfg.verbose) + results |> emplace(r) + } + } else { + to_log(LOG_INFO, "[RUN ] lanes, concurrently: {join(lanes, ", ")}\n") + let lane_results <- run_lanes(ctx, cfg, lanes) + for (r in lane_results) { + report_gate(r, cfg.verbose) + } + results |> push_from(lane_results) + } + } var n_pass = 0 var n_fail = 0 diff --git a/utils/internal/preflight/tests/test_changed_set.das b/utils/internal/preflight/tests/test_changed_set.das index bdf6683c87..a59aaf25b0 100644 --- a/utils/internal/preflight/tests/test_changed_set.das +++ b/utils/internal/preflight/tests/test_changed_set.das @@ -2,6 +2,7 @@ options gen2 options indenting = 4 require dastest/testing_boost public +require daslib/strings_boost require ../config.das @@ -14,3 +15,109 @@ def test_has_path_under(t : T?) { t |> success(!has_path_under(none, prefix), "an empty diff does not") t |> success(has_path_under(["modules/dasLLAMA/REVIEW.md\r"], prefix), "a CR-terminated line still counts") } + +// A gate's reach: no reach set means the gate runs whatever changed; a reach set is hit by its own +// prefixes or by the core, and missed by everything else. +[test] +def test_reach_hit(t : T?) { + let none : array + let imgui <- ["modules/dasImgui/", "modules/dasGlfw/"] + t |> success(reach_hit(["plans/x.md"], none), "an empty reach set runs on any change") + t |> success(reach_hit(["modules/dasImgui/daslib/imgui_boost.das"], imgui), "a path under the gate's own prefix reaches it") + t |> success(reach_hit(["src/ast/ast_parse.cpp"], imgui), "a core change reaches every gate") + t |> success(reach_hit(["CMakeLists.txt"], imgui), "the top-level CMakeLists is core") + t |> success(!reach_hit(["modules/dasLLAMA/tests/run.das", "plans/x.md"], imgui), "a change elsewhere misses the gate") + t |> success(!reach_hit(none, imgui), "an empty diff misses every reach set") +} + +// The compile sweep's root filter: program roots under the swept trees, never tests, generated files, +// fixtures, or a module tree nested inside an example through a junction. +[test] +def test_is_sweep_root_path(t : T?) { + t |> success(is_sweep_root_path("utils/lint/main.das"), "a tool's main is a root candidate") + t |> success(is_sweep_root_path("modules/dasImgui/examples/features/active_widget.das"), "a module's own example is a candidate") + t |> success(is_sweep_root_path("tutorials/language/01_hello.das"), "a tutorial is a candidate") + t |> success(!is_sweep_root_path("utils/lint/tests/lint006_division_by_zero.das"), "a tests folder is not swept") + t |> success(!is_sweep_root_path("examples/x/test_thing.das"), "a test_ file is not swept") + t |> success(!is_sweep_root_path("examples/x/_fixture.das"), "an underscore fixture is not swept") + t |> success(!is_sweep_root_path("tutorials/_aot_generated/x.das"), "generated files are not swept") + t |> success(!is_sweep_root_path("examples/daStrudel/sfx_lab/modules/dasImgui/bind/bind_imgui.das"), "a module tree reached through an example's junction is not swept") + t |> success(!is_sweep_root_path("utils/lint/README.md"), "only .das files are candidates") +} + +// A lane child's report: the verdict lines with their seconds, details and indented breakdown; RUN +// lines and prose are not verdicts; a docs child reports several gates. +[test] +def test_parse_gate_reports(t : T?) { + let out = "preflight: fast tier; daslang=bin/daslang\n[RUN ] tests-jit\n[PASS] tests-jit (396.2s) — JIT suite ok\n Top 3 slowest files:\n 12.0s tests/a.das\n\npreflight: 1 passed\n" + let one <- parse_gate_reports(out) + t |> equal(length(one), 1, "one verdict line") + t |> equal(one[0].tag, "PASS", "the tag") + t |> equal(one[0].name, "tests-jit", "the gate name, without the seconds") + t |> success(one[0].seconds > 396.0lf && one[0].seconds < 396.4lf, "the seconds") + t |> equal(one[0].detail, "JIT suite ok", "the detail after the dash") + t |> equal(one[0].breakdown, "Top 3 slowest files:\n12.0s tests/a.das", "the indented breakdown, trimmed") + let docs = "[RUN ] docs\n[PASS] das2rst (12.0s)\n[SKIP] sphinx — sphinx-build not found\n[FAIL] stubs (0.3s) — 2 stub(s) unfilled\n" + let many <- parse_gate_reports(docs) + t |> equal(length(many), 3, "a docs child reports each sub-gate") + t |> equal(many[1].tag, "SKIP", "a skip without seconds parses") + t |> equal(many[1].name, "sphinx", "its name") + t |> equal(many[1].detail, "sphinx-build not found", "its reason") + t |> equal(many[2].tag, "FAIL", "a fail parses") + t |> equal(length(parse_gate_reports("died before reporting\n")), 0, "no verdict line yields no report") + let logged = "[I] [RUN ] tests-cpp\n[I] [PASS] tests-cpp (20.1s)\n[I] Top 1 slowest files:\n[I] 2.0s tests/b.das\n[I] \n[W] [SKIP] imgui — no display\n" + let tagged <- parse_gate_reports(logged) + t |> equal(length(tagged), 2, "a to_log level prefix on every captured line is not the verdict tag") + t |> equal(tagged[0].name, "tests-cpp", "the gate name behind the [I] prefix") + t |> equal(tagged[0].breakdown, "Top 1 slowest files:\n2.0s tests/b.das", "the breakdown behind the prefix, trimmed") + t |> equal(tagged[1].tag, "SKIP", "a warning-level skip line parses") + // the edges: a whitespace-only indented line ends the breakdown (a blank line does), a `[` line with no + // `]` is prose, a name with ` (` that is not a seconds group keeps it, and a child cut off mid-breakdown + // still closes its report + let edges = "[PASS] lint (2.0s)\n Top 1 slowest files:\n \n 1.0s x.das\n[bracket without close\n[PASS] docs (sphinx) — ok\n[FAIL] tests-jit (9.5s)\n 12.0s tests/z.das" + let e <- parse_gate_reports(edges) + t |> equal(length(e), 3, "three verdicts") + t |> equal(e[0].breakdown, "Top 1 slowest files:", "a whitespace-only indented line ends the breakdown, not a blank row in it") + t |> equal(e[1].name, "docs (sphinx)", "a parenthesis that is not a seconds group stays in the name") + t |> success(e[1].seconds == 0.0lf, "no seconds parsed from it") + t |> equal(e[2].breakdown, "12.0s tests/z.das", "the last report closes without a trailing newline") +} + +[test] +def test_tier_runs(t : T?) { + t |> success(tier_runs("fast", false), "the fast tier runs without --full") + t |> success(tier_runs("fast", true), "and with it") + t |> success(!tier_runs("lane", false), "a lane waits for --full") + t |> success(tier_runs("lane", true), "and runs under it") + t |> success(!tier_runs("module", false) && !tier_runs("module", true), "a module gate never runs from a tier") +} + +[test] +def test_sweep_excluded(t : T?) { + t |> success(sweep_excluded("examples/crash/crash.das"), "an excluded tree's root") + t |> success(sweep_excluded("utils/internal/ast-fuzz/selftest/broken.das"), "the fuzz fixtures") + t |> success(!sweep_excluded("examples/crashes/x.das"), "a prefix match is on the whole path segment") + t |> success(!sweep_excluded("utils/lint/main.das"), "a swept tool root") + for (why in values(SWEEP_EXCLUDED)) { + t |> success(!empty(why), "every exclusion carries its reason") + } +} + +[test] +def test_sweep_root_text(t : T?) { + t |> success(is_sweep_root_text("options gen2\n[export]\ndef main() \{\n\}\n"), "an exported main is a root") + t |> success(!is_sweep_root_text("options gen2\ndef helper() \{\n\}\n"), "a module without main is not") + t |> success(!is_sweep_root_text("def main() \{\n\}\n"), "an unexported main is not a program root") + t |> success(is_heavy_sweep_root("require dasllama/dasllama\n[export]\ndef main() \{\}\n"), "an engine require is heavy") + t |> success(!is_heavy_sweep_root("require daslib/fio\n[export]\ndef main() \{\}\n"), "a plain root is light") +} + +[test] +def test_failed_files_block(t : T?) { + let one = "13205 tests, 13201 passed, 1 failed\n\nFAILURES:\n tests/json/test_sscan_json.das — 1 failed, 0 errors\n\nFAILED! (109.3s)\n" + t |> equal(failed_files_block(one), "tests/json/test_sscan_json.das", "the file before the dash") + t |> equal(failed_files_block("all green\n"), "the suite (no FAILURES block in the output)", "no block, a stated default") + let rows <- [for (i in range(7)); " tests/t{i}.das — 1 failed"] + let many = "FAILURES:\n" + join(rows, "\n") + "\n" + t |> equal(failed_files_block(many), "tests/t0.das, tests/t1.das, tests/t2.das, tests/t3.das, tests/t4.das, and 2 more", "five named, the rest counted") +}