From 696d870b8f47f812bb4558436295a493be5a28e6 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Sat, 22 Aug 2026 20:28:01 +0100 Subject: [PATCH] ci: smoke both iOS flavors on a real device real-device-smoke-ios fetched the nodejs-mobile-ios artifact -- the full flavor -- and lite's only device-class coverage was curated-tests-ios on a simulator. A simulator is a macOS process and is not subject to the address-space limits a real iOS process is, so that pairing cannot tell you whether a flavor boots on hardware. 24.19.0-1 shipped an iOS lite build that aborts during Isolate init on every physical device with every simulator leg green. Give the iOS leg a flavor matrix, defaulting to both, in the shape ios-simulator-tests.yml already uses: an ios_flavors JSON-array input, fromJSON'd into strategy.matrix, and FLAVOR_SUFFIX selecting the per-flavor artifact. fail-fast is off because which flavors boot is the signal, and the build tag carries the flavor so the two BrowserStack builds are distinguishable. This only costs device minutes on release and dry-run pushes, where the job is already gated -- it never runs on a PR. Android stays full-only; its lite flavor has the same blind spot but the emulator is a closer proxy there, and widening it is a separate call. Publish already needs: this job, so a lite build that cannot start on hardware now blocks the release instead of shipping. --- .github/workflows/browserstack-smoke.yml | 36 ++++++++++++++++++++++-- docs/RELEASING.md | 6 ++-- docs/TESTING.md | 8 ++++-- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/.github/workflows/browserstack-smoke.yml b/.github/workflows/browserstack-smoke.yml index 2b6bc8296f..2c105bc8f8 100644 --- a/.github/workflows/browserstack-smoke.yml +++ b/.github/workflows/browserstack-smoke.yml @@ -5,6 +5,13 @@ name: BrowserStack device smoke # arm64 device (Espresso) and a physical iPhone (XCUITest). Checks the device # loader's dlopen path and 16 KB-page hardware. # +# The iOS leg runs once per flavor (`ios_flavors`). A simulator is a macOS +# process and does not share the address-space limits a real iOS process is +# subject to, so a flavor that only ever runs on the simulator is untested +# against the constraint most likely to stop it booting — which is how a lite +# build that aborts during Isolate init on every physical device shipped in +# 24.19.0-1. Simulator green is not evidence about device behaviour. +# # Reuses the libnode/xcframework a green Build run (build.yml) already # produced for the target commit — nothing is compiled here except the tiny # testnode app, addon, and test bundles. @@ -21,6 +28,10 @@ on: description: "BrowserStack iOS devices (CSV)" required: false default: "iPhone 15-17" + ios_flavors: + description: 'iOS flavors to smoke (JSON array, e.g. ["full","lite"])' + required: false + default: '["full","lite"]' platform: description: "Which platform legs to run" required: false @@ -44,6 +55,14 @@ on: type: string required: false default: both + # JSON array, mirroring ios-simulator-tests.yml's `flavors`. Both by + # default: this workflow is release-gated, so there is no PR-cost case + # for trimming it, and the flavor a release ships is exactly the one + # that has to be proven on hardware. + ios_flavors: + type: string + required: false + default: '["full","lite"]' secrets: BROWSERSTACK_USER: required: true @@ -142,12 +161,21 @@ jobs: ios-device-smoke: # Exclusion, not inclusion — see android-device-smoke. if: inputs.platform != 'android' + # fail-fast off: which flavors boot on hardware is the signal, so let both + # report rather than cancelling the survivor. + strategy: + fail-fast: false + matrix: + flavor: ${{ fromJSON(inputs.ios_flavors || '["full","lite"]') }} # macos-15 (Xcode 16.x), NOT macos-26: an XCUITest runner built by Xcode 26 # references XCTest symbols (e.g. XCTCommandLineToolHelper) that don't # exist in the on-device XCTest of BrowserStack's iOS 15-18 fleet — dyld # aborts the runner before any test executes. runs-on: macos-15 timeout-minutes: 60 + env: + NODEJS_MOBILE_FLAVOR: ${{ matrix.flavor }} + FLAVOR_SUFFIX: ${{ matrix.flavor == 'lite' && '-lite' || '' }} steps: - uses: actions/checkout@v7 with: @@ -157,7 +185,7 @@ jobs: - name: Fetch NodeMobile.xcframework from the Build run uses: ./.github/actions/fetch-build-artifact with: - name: nodejs-mobile-ios + name: nodejs-mobile-ios${{ env.FLAVOR_SUFFIX }} dest: nm-ios build_run_id: ${{ inputs.build_run_id }} token: ${{ github.token }} @@ -167,6 +195,8 @@ jobs: mkdir -p out_ios rm -rf out_ios/NodeMobile.xcframework cp -R nm-ios/NodeMobile.xcframework out_ios/NodeMobile.xcframework + test -d out_ios/NodeMobile.xcframework \ + || { echo "::error::NodeMobile.xcframework missing from nodejs-mobile-ios${FLAVOR_SUFFIX}"; ls -R nm-ios || true; exit 1; } - name: Build crc-native addon (ios-arm64 device) run: | @@ -219,7 +249,9 @@ jobs: run: | DEVICES="$DEVICES_INPUT" [ -n "$DEVICES" ] || DEVICES="$DEFAULT_IOS_DEVICES" + # Flavor in the build tag: both legs upload to the same BrowserStack + # project, and an untagged pair is indistinguishable in the dashboard. ./tools/mobile-test/browserstack/run-bs-build.sh xcuitest \ tools/mobile-test/ios/testnode/testnode.ipa \ tools/mobile-test/ios/testnode/testnodeUITests.zip \ - "$DEVICES" nodejs-mobile "sha-${{ github.sha }}" + "$DEVICES" nodejs-mobile "sha-${{ github.sha }}-${{ matrix.flavor }}" diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 6ebfbe7851..2d4a213718 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -50,9 +50,9 @@ Releasing is a button, a review, and (optionally) an approval: 5. Publish tags **`vX.Y.Z-R`** on a materialized full-source commit (the release stays browsable as a complete tree) and creates the GitHub **prerelease** with four zips: `nodejs-mobile-{android,ios}{,-lite}-X.Y.Z-R.zip`. - Promote (untick "prerelease") when satisfied — the `full` flavor has - already passed real devices by construction; `lite` is - emulator/simulator-tested only. + Promote (untick "prerelease") when satisfied — both iOS flavors have + passed a real device by construction, as has Android `full`; Android + `lite` is still emulator-tested only. A failed gate means no tag and no release; fix on `recipe` and the next push retries automatically (the version is still untagged — the trigger is diff --git a/docs/TESTING.md b/docs/TESTING.md index 9a98e5087e..14b23fb1bc 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -68,7 +68,7 @@ it through `tools/test.py`. | `build.yml` → `smoke-{android,ios}` (+ the NAPI symbol assert in `combine-android`) | ubuntu+KVM / macos | PR · push `recipe` | **boot smoke**: the exact shipping artifact boots and runs JS; NAPI symbols in `.dynsym` | | `build.yml` → `curated-tests-android` / `curated-tests-ios` | ubuntu+KVM / macos | PR · push `recipe` · releases | **curated device tests**: the curated subset + crc-native addon load on an x86_64 emulator and arm64 simulator | | `full-device-suite.yml` (also `build.yml` → `full-suite-android` / `full-suite-ios` on releases and on PRs that bump `upstream-base.txt`) | ubuntu+KVM / macos | nightly 03:00 UTC · dispatch · releases · upstream-bump PRs | **full device suite**: the whole non-`.status`-skipped `test/parallel` + `test/sequential` suite on both platforms, 4 round-robin shards each (`test.py --run=n,4`). The curated gate covers what someone chose; this covers everything else, so a test upstream adds tomorrow is picked up without anyone noticing it exists. On a PR that moves the pinned tag it also gates `ci-required` (via `full-suite-gate`): an upgrade carries whatever tests upstream added, and an allow-list gate cannot see them | -| `build.yml` → `real-device-smoke-android` / `real-device-smoke-ios` | ubuntu / macos-15 + BrowserStack | releases (untagged version of record; required to publish) · dispatch | **real-device smoke**: boot + crc-native addon load on physical devices — Android arm64 (Pixel 9, 16 KB pages) via Espresso and iPhone via XCUITest. Needs `BROWSERSTACK_USER`/`BROWSERSTACK_PW` secrets. | +| `build.yml` → `real-device-smoke-android` / `real-device-smoke-ios` | ubuntu / macos-15 + BrowserStack | releases (untagged version of record; required to publish) · dispatch | **real-device smoke**: boot + crc-native addon load on physical devices — Android arm64 (Pixel 9, 16 KB pages) via Espresso and iPhone via XCUITest. The iOS leg runs **once per flavor** (`ios_flavors`, both by default): a simulator is a macOS process and is not subject to the address-space limits a real iOS process is, so simulator-only coverage cannot tell you a flavor boots on hardware. Android is still full-only. Needs `BROWSERSTACK_USER`/`BROWSERSTACK_PW` secrets. | Every job first **materializes** the source tree from the recipe branch (`.github/actions/materialize` runs `scripts/prepare.sh` and verifies the @@ -102,7 +102,11 @@ it; don't merge through a red one without knowing why it's red. They are blocking on the release chain, where `publish` `needs:` them. The real-device smoke stays release-only: GitHub Actions minutes are free for -this project, BrowserStack device minutes are not. +this project, BrowserStack device minutes are not. Within that budget the iOS +leg still runs per flavor rather than for `full` alone — the flavor a consumer +ships is the one that has to be proven on hardware, and 24.19.0-1 shipped an +iOS `lite` build that aborted during `Isolate` init on every physical device +while every simulator leg stayed green. ### The curated gate and the full suite