diff --git a/docs/benchmarking-workflow-dotnet-runtime.md b/docs/benchmarking-workflow-dotnet-runtime.md index f5f8f874f46..966c073fa3e 100644 --- a/docs/benchmarking-workflow-dotnet-runtime.md +++ b/docs/benchmarking-workflow-dotnet-runtime.md @@ -255,6 +255,20 @@ recorded in that SDK's `Microsoft.NETCoreSdk.BundledVersions.props`. `PERFLAB_WASM_PACKAGE_VERSION` should remain unset; that override is reserved for non-official runtime payloads. +`--wasm-ready-to-run` publishes per-assembly ReadyToRun images. Replace it with +`--wasm-ready-to-run-composite` to compile the benchmark closure into a single +composite image (`PublishReadyToRunComposite=true`, published as +`.r2r.wasm` and loaded through the boot config's `coreAssembly` +resources). Composite mode requires a WebAssembly SDK with composite support +(dotnet/runtime#134618) and ReadyToRun SDK tasks with wasm output naming +(dotnet/sdk#56395). Until that SDK change is available, set +`PERFLAB_WASM_CROSSGEN2_TASKS_DIR` to a directory containing dotnet/runtime's +`Crossgen2Tasks` shim (`artifacts/bin/Crossgen2Tasks//` from a +runtime build, or `staging/Crossgen2Tasks/` in the `BrowserWasmCoreCLR` perf +artifact); composite runs then pass it to the WebAssembly SDK through +`Crossgen2SdkOverridePropsPath`/`Crossgen2SdkOverrideTargetsPath`. CI reports +the two modes as `R2RType=r2r` and `R2RType=r2r_composite`. + #### Note about "file ... being used by another process" error If you are seeing warnings like: diff --git a/eng/pipelines/runtime-wasm-perf-jobs.yml b/eng/pipelines/runtime-wasm-perf-jobs.yml index 41608199247..97eb4e79843 100644 --- a/eng/pipelines/runtime-wasm-perf-jobs.yml +++ b/eng/pipelines/runtime-wasm-perf-jobs.yml @@ -158,8 +158,7 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - # Run CoreCLR WASM R2R microbenchmarks using the same runtime payload. - # Browser WASM supports per-assembly R2R rather than composite R2R. + # Run CoreCLR WASM per-assembly R2R microbenchmarks using the same runtime payload. - ${{ if not(startswith(variables['Build.SourceBranch'], 'refs/heads/release')) }}: - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: @@ -182,3 +181,29 @@ jobs: performanceRepoAlias: ${{ parameters.performanceRepoAlias }} ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} + + # Run CoreCLR WASM composite R2R microbenchmarks using the same runtime payload. + # The whole benchmark closure is compiled into one .r2r.wasm image (PublishReadyToRunComposite=true) + # and reported as R2RType=r2r_composite, a history separate from per-assembly R2RType=r2r. + - ${{ if not(startswith(variables['Build.SourceBranch'], 'refs/heads/release')) }}: + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: wasm_coreclr + codeGenType: 'wasm' + r2rRunType: 'r2r_composite' + runKind: micro + logicalMachine: 'perfviper' + javascriptEngine: 'v8' + additionalJobIdentifier: coreclr_r2r_composite_v8 + downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml index 80a7f771cde..5f96e5d0c89 100644 --- a/eng/pipelines/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -50,7 +50,7 @@ parameters: isCompare: false # optional -- If the job is comparing two configurations onlySanityCheck: false # optional -- Whether to only run a sanity check for pipeline validation, not performance testing pgoRunType: '' # optional -- PGO run type (e.g. nodynamicpgo) - r2rRunType: '' # optional -- R2R run type (e.g. nor2r) + r2rRunType: '' # optional -- R2R run type (e.g. nor2r, r2r, r2r_composite) experimentName: '' # optional -- Name of the experiment javascriptEngine: '' # optional -- JavaScript engine to use iOSLlvmBuild: false # optional -- Whether to build iOS with LLVM @@ -242,7 +242,7 @@ jobs: - '--cross-build' - ${{ if ne(parameters.additionalSetupParameters, '') }}: - '${{ parameters.additionalSetupParameters }}' - - ${{ if and(ne(parameters.wasmWorkloadSource, ''), eq(parameters.runtimeType, 'wasm_coreclr'), eq(parameters.r2rRunType, 'r2r')) }}: + - ${{ if and(ne(parameters.wasmWorkloadSource, ''), eq(parameters.runtimeType, 'wasm_coreclr'), in(parameters.r2rRunType, 'r2r', 'r2r_composite')) }}: - '--wasm-workload-source "${{ parameters.wasmWorkloadSource }}"' - template: /eng/pipelines/templates/send-to-helix-step.yml parameters: diff --git a/scripts/benchmarks_ci.py b/scripts/benchmarks_ci.py index 17131362e9e..b69d9a8a9dc 100755 --- a/scripts/benchmarks_ci.py +++ b/scripts/benchmarks_ci.py @@ -231,10 +231,11 @@ def main(argv: list[str]): if args.wasm_workload_source and not ( args.wasm and args.wasm_runtime_flavor == 'CoreCLR' - and args.wasm_ready_to_run): + and micro_benchmarks.is_wasm_ready_to_run(args)): raise ValueError( '--wasm-workload-source requires --wasm ' - '--wasm-runtime-flavor CoreCLR --wasm-ready-to-run') + '--wasm-runtime-flavor CoreCLR --wasm-ready-to-run ' + '(or --wasm-ready-to-run-composite)') # Acquire necessary tools (dotnet) if not args.dotnet_path: diff --git a/scripts/build_runtime_payload.py b/scripts/build_runtime_payload.py index 926b7efcfbd..f3669493baf 100644 --- a/scripts/build_runtime_payload.py +++ b/scripts/build_runtime_payload.py @@ -22,6 +22,7 @@ "build_r2r_interpreter_payload", "build_wasm_payload", "build_wasm_coreclr_payload", + "WASM_CROSSGEN2_TASKS_PAYLOAD_DIR", ] @@ -322,6 +323,55 @@ def build_wasm_payload( _set_permissions_recursive([wasm_dotnet_dir, wasm_built_nugets_dir], mode=0o664) # rw-rw-r-- +# dotnet/runtime's wasm-aware ReadyToRun SDK task shim (Crossgen2Tasks.dll plus the +# Microsoft.NET.CrossGen.props/.targets that load it). Composite CoreCLR WASM R2R needs it until +# dotnet/sdk#56395 reaches the SDK shipped in the BrowserWasmCoreCLR artifact. +WASM_CROSSGEN2_TASKS_PAYLOAD_DIR = "crossgen2-tasks" +WASM_CROSSGEN2_TASKS_FILES = ( + "Crossgen2Tasks.dll", + "Microsoft.NET.CrossGen.props", + "Microsoft.NET.CrossGen.targets", +) + + +def _stage_wasm_crossgen2_tasks( + browser_wasm_coreclr_archive_or_dir: str, + payload_parent_dir: str, +) -> bool: + """Copy the optional ``staging/Crossgen2Tasks`` shim into the payload. + + Returns True when the shim was staged, False when the artifact predates it + (no ``staging/Crossgen2Tasks`` content at all). A shim that is present but + missing any required file raises ``ValueError``. + """ + dest_dir = os.path.join(payload_parent_dir, WASM_CROSSGEN2_TASKS_PAYLOAD_DIR) + if (os.path.isdir(browser_wasm_coreclr_archive_or_dir) + and not os.path.isdir(os.path.join(browser_wasm_coreclr_archive_or_dir, "staging", "Crossgen2Tasks"))): + getLogger().info("BrowserWasmCoreCLR artifact does not contain the Crossgen2Tasks shim") + return False + + extract_archive_or_copy( + browser_wasm_coreclr_archive_or_dir, dest_dir, prefix="staging/Crossgen2Tasks/" + ) + # Archives can't be probed for a folder up front; nothing extracted means no shim. + if (not os.path.isdir(browser_wasm_coreclr_archive_or_dir) + and not any(path.is_file() for path in Path(dest_dir).rglob("*"))): + shutil.rmtree(dest_dir, ignore_errors=True) + getLogger().info("BrowserWasmCoreCLR artifact does not contain the Crossgen2Tasks shim") + return False + + missing = [ + name for name in WASM_CROSSGEN2_TASKS_FILES + if not os.path.isfile(os.path.join(dest_dir, name)) + ] + if missing: + raise ValueError( + f"Incomplete Crossgen2Tasks shim in BrowserWasmCoreCLR artifact, missing: {', '.join(missing)}") + + getLogger().info("Staged Crossgen2Tasks shim into %s", dest_dir) + return True + + def build_wasm_coreclr_payload( browser_wasm_coreclr_archive_or_dir: str, payload_parent_dir: str, @@ -331,7 +381,8 @@ def build_wasm_coreclr_payload( This is a self-contained payload for running CoreCLR WASM benchmarks without requiring Mono artifacts. The archive/directory layout is expected to contain a `staging/` folder with `dotnet-none` (SDK) and - `microsoft.netcore.app.runtime.browser-wasm` (CoreCLR runtime pack) subfolders. + `microsoft.netcore.app.runtime.browser-wasm` (CoreCLR runtime pack) subfolders, + and optionally `Crossgen2Tasks` (staged as ``WASM_CROSSGEN2_TASKS_PAYLOAD_DIR``). Returns: The shared version of the locally built WebAssembly SDK and Crossgen2 packages. @@ -351,6 +402,8 @@ def build_wasm_coreclr_payload( ) local_package_version = _get_wasm_local_package_version(wasm_built_nugets_dir) _ensure_wasm_tool_framework_version(wasm_dotnet_dir, local_package_version) + has_crossgen2_tasks = _stage_wasm_crossgen2_tasks( + browser_wasm_coreclr_archive_or_dir, payload_parent_dir) # Determine version from the runtime pack directory structure runtime_pack_src = os.path.join( @@ -377,7 +430,10 @@ def build_wasm_coreclr_payload( else: getLogger().warning("Microsoft.NETCore.App.Ref pack not found – cannot determine version") - _set_permissions_recursive([wasm_dotnet_dir, wasm_built_nugets_dir], mode=0o664) + permission_dirs = [wasm_dotnet_dir, wasm_built_nugets_dir] + if has_crossgen2_tasks: + permission_dirs.append(os.path.join(payload_parent_dir, WASM_CROSSGEN2_TASKS_PAYLOAD_DIR)) + _set_permissions_recursive(permission_dirs, mode=0o664) return local_package_version diff --git a/scripts/micro_benchmarks.py b/scripts/micro_benchmarks.py index b98bf4012c1..96e4936c77e 100755 --- a/scripts/micro_benchmarks.py +++ b/scripts/micro_benchmarks.py @@ -155,7 +155,17 @@ def __get_bdn_arguments(user_input: str) -> list[str]: required=False, default=False, action='store_true', - help='Publish CoreCLR WASM benchmarks as ReadyToRun' + help='Publish CoreCLR WASM benchmarks as per-assembly ReadyToRun' + ) + + parser.add_argument( + '--wasm-ready-to-run-composite', + dest='wasm_ready_to_run_composite', + required=False, + default=False, + action='store_true', + help='Publish CoreCLR WASM benchmarks as a single composite ' + 'ReadyToRun image (implies ReadyToRun)' ) parser.add_argument( @@ -252,9 +262,21 @@ def __process_arguments(args: list[str]): return parsed_args +def is_wasm_ready_to_run_composite(args: Any) -> bool: + return bool(getattr(args, 'wasm_ready_to_run_composite', False)) + + +def is_wasm_ready_to_run(args: Any) -> bool: + '''True for either per-assembly or composite CoreCLR WASM ReadyToRun.''' + return bool(args.wasm_ready_to_run) or is_wasm_ready_to_run_composite(args) + + def validate_wasm_ready_to_run(args: Any) -> None: - if args.wasm_ready_to_run and (not args.wasm or args.wasm_runtime_flavor != 'CoreCLR'): - raise ArgumentTypeError('--wasm-ready-to-run requires --wasm --wasm-runtime-flavor CoreCLR') + if not args.wasm or args.wasm_runtime_flavor != 'CoreCLR': + if is_wasm_ready_to_run_composite(args): + raise ArgumentTypeError('--wasm-ready-to-run-composite requires --wasm --wasm-runtime-flavor CoreCLR') + if args.wasm_ready_to_run: + raise ArgumentTypeError('--wasm-ready-to-run requires --wasm --wasm-runtime-flavor CoreCLR') def configure_wasm_ready_to_run(args: Any) -> None: @@ -263,7 +285,51 @@ def configure_wasm_ready_to_run(args: Any) -> None: # BenchmarkDotNet builds generated projects in child processes. MSBuild # imports environment variables as properties, which lets the generated # WASM project opt into R2R without requiring a new BDN command-line option. - environ['PERFLAB_WASM_READY_TO_RUN'] = str(args.wasm_ready_to_run).lower() + # Both variables are always written so a stale value from the parent + # environment cannot select a different mode. + environ['PERFLAB_WASM_READY_TO_RUN'] = str(is_wasm_ready_to_run(args)).lower() + environ['PERFLAB_WASM_READY_TO_RUN_COMPOSITE'] = str(is_wasm_ready_to_run_composite(args)).lower() + configure_wasm_crossgen2_sdk_override(args) + + +WASM_CROSSGEN2_TASKS_DIR_VARIABLE = 'PERFLAB_WASM_CROSSGEN2_TASKS_DIR' +WASM_CROSSGEN2_SDK_OVERRIDE_VARIABLES = ( + 'Crossgen2SdkOverridePropsPath', + 'Crossgen2SdkOverrideTargetsPath', +) + + +def configure_wasm_crossgen2_sdk_override(args: Any) -> None: + '''Activate dotnet/runtime's wasm-aware ReadyToRun task shim for composite R2R. + + Composite CoreCLR WASM R2R needs SDK ReadyToRun tasks that name the owner + image .r2r.wasm (dotnet/sdk#56395). Until that reaches the SDK in + use, the CoreCLR WASM payload ships the shim and exports its directory in + PERFLAB_WASM_CROSSGEN2_TASKS_DIR. The WebAssembly SDK reads the override + paths during props evaluation, before BenchmarkDotNet imports + MicroBenchmarks.Wasm.targets, so they are passed as environment + properties. TODO: remove with dotnet/runtime#135023. + + Inherited override paths are always cleared first so per-assembly R2R + keeps the SDK's tasks and composite never uses a stale shim; set + PERFLAB_WASM_CROSSGEN2_TASKS_DIR to select a shim. + ''' + for variable in WASM_CROSSGEN2_SDK_OVERRIDE_VARIABLES: + environ.pop(variable, None) + + tasks_dir = environ.get(WASM_CROSSGEN2_TASKS_DIR_VARIABLE) + if not is_wasm_ready_to_run_composite(args) or not tasks_dir: + return + + props_path = path.join(tasks_dir, 'Microsoft.NET.CrossGen.props') + targets_path = path.join(tasks_dir, 'Microsoft.NET.CrossGen.targets') + for required in (path.join(tasks_dir, 'Crossgen2Tasks.dll'), props_path, targets_path): + if not path.isfile(required): + raise FileNotFoundError( + f'{WASM_CROSSGEN2_TASKS_DIR_VARIABLE} does not contain {path.basename(required)}: {tasks_dir}') + + environ['Crossgen2SdkOverridePropsPath'] = props_path + environ['Crossgen2SdkOverrideTargetsPath'] = targets_path def __get_benchmarkdotnet_arguments(framework: str, args: Any) -> list[str]: diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index f3b8393240f..83a183dbcac 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -226,7 +226,8 @@ def get_pre_commands( build_config: str, v8_version: str, wasm_local_package_version: Optional[str] = None, - wasm_workload_source: Optional[str] = None): + wasm_workload_source: Optional[str] = None, + wasm_crossgen2_tasks_dir: Optional[str] = None): if os_group == "windows" and runtime_type in ("wasm", "wasm_coreclr"): raise ValueError( "WASM performance job prerequisite setup is not supported on Windows") @@ -317,6 +318,13 @@ def get_pre_commands( "PERFLAB_WASM_PACKAGE_VERSION", wasm_local_package_version) ] + if wasm_crossgen2_tasks_dir: + install_prerequisites += [ + set_shell_environment_variable( + os_group, + "PERFLAB_WASM_CROSSGEN2_TASKS_DIR", + wasm_crossgen2_tasks_dir) + ] elif not use_workload_source: raise ValueError( "CoreCLR WASM requires either a private runtime payload or " @@ -697,6 +705,9 @@ def get_run_configurations( configurations["R2RType"] = "nor2r" elif r2r_run_type == "r2r": configurations["R2RType"] = "r2r" + elif r2r_run_type == "r2r_composite": + # CoreCLR WASM composite R2R gets its own PerfLab history, distinct from per-assembly r2r. + configurations["R2RType"] = "r2r_composite" if runtime_type == "coreclr_r2r_interpreter": configurations["R2RType"] = "r2r_interpreter" @@ -754,7 +765,8 @@ def get_work_item_command( wasm_coreclr: bool = False, wasm_ready_to_run: bool = False, only_sanity_check: bool = False, - wasm_workload_source: Optional[str] = None): + wasm_workload_source: Optional[str] = None, + wasm_ready_to_run_composite: bool = False): if os_group == "windows": work_item_command = [ "python", @@ -793,9 +805,11 @@ def get_work_item_command( ] if wasm_coreclr: work_item_command += ["--wasm-runtime-flavor", "CoreCLR"] - if wasm_ready_to_run: + if wasm_ready_to_run_composite: + work_item_command += ["--wasm-ready-to-run-composite"] + elif wasm_ready_to_run: work_item_command += ["--wasm-ready-to-run"] - if wasm_ready_to_run and wasm_workload_source: + if (wasm_ready_to_run or wasm_ready_to_run_composite) and wasm_workload_source: work_item_command += [ "--wasm-workload-source", wasm_workload_source, @@ -861,6 +875,8 @@ def run_performance_job(args: RunPerformanceJobArgs): wasm_coreclr = args.runtime_type == "wasm_coreclr" wasm = args.runtime_type == "wasm" or wasm_coreclr # wasm_coreclr also uses wasm infrastructure wasm_aot = wasm and is_aot and not wasm_coreclr + if args.r2r_run_type == "r2r_composite" and not wasm_coreclr: + raise Exception("r2r_composite R2R run type is only supported for the wasm_coreclr runtime type") working_dir = os.path.join(args.performance_repo_dir, "CorrelationStaging") # folder in which the payload and workitem directories will be made work_item_dir = os.path.join(working_dir, "workitem", "") # Folder in which the work item commands will be run in @@ -1021,11 +1037,12 @@ def run_performance_job(args: RunPerformanceJobArgs): v8_version = "" wasm_local_package_version = None + helix_wasm_crossgen2_tasks_dir = None wasm_workload_source = normalize_wasm_workload_source( args.wasm_workload_source) wasm_sdk_cohort = ( wasm_coreclr - and args.r2r_run_type == "r2r" + and args.r2r_run_type in ("r2r", "r2r_composite") and wasm_workload_source is not None) helix_wasm_workload_source = ( wasm_workload_source if wasm_sdk_cohort else None) @@ -1054,6 +1071,18 @@ def run_performance_job(args: RunPerformanceJobArgs): browser_wasm_coreclr_dir, payload_dir, ) + if args.r2r_run_type == "r2r_composite": + # Composite needs the artifact's wasm-aware ReadyToRun task shim until + # dotnet/sdk#56395 reaches the artifact's SDK (dotnet/runtime#135023). + if os.path.isdir(os.path.join(payload_dir, WASM_CROSSGEN2_TASKS_PAYLOAD_DIR)): + helix_wasm_crossgen2_tasks_dir = ( + f"%HELIX_CORRELATION_PAYLOAD%\\{WASM_CROSSGEN2_TASKS_PAYLOAD_DIR}" + if args.os_group == "windows" + else f"$HELIX_CORRELATION_PAYLOAD/{WASM_CROSSGEN2_TASKS_PAYLOAD_DIR}") + else: + getLogger().warning( + "BrowserWasmCoreCLR artifact has no Crossgen2Tasks shim; composite R2R " + "will use the SDK's ReadyToRun tasks") elif wasm and not wasm_coreclr: if args.libraries_download_dir is None: @@ -1247,7 +1276,8 @@ def run_performance_job(args: RunPerformanceJobArgs): args.build_config, v8_version, wasm_local_package_version, - helix_wasm_workload_source) + helix_wasm_workload_source, + helix_wasm_crossgen2_tasks_dir) helix_post_commands = get_post_commands(args.os_group, args.internal, args.runtime_type) # Point ML.NET at the SSWE model that was pre-downloaded into the correlation payload above, so it @@ -1497,7 +1527,8 @@ def get_work_item_command_for_artifact_dir(artifact_dir: str): wasm_coreclr, wasm_coreclr and args.r2r_run_type == "r2r", args.only_sanity_check, - helix_wasm_workload_source) + helix_wasm_workload_source, + wasm_coreclr and args.r2r_run_type == "r2r_composite") work_item_command = get_work_item_command_for_artifact_dir(bdn_artifacts_directory) baseline_work_item_command = get_work_item_command_for_artifact_dir(bdn_baseline_artifacts_dir) diff --git a/scripts/tests/test_wasm_coreclr_r2r.py b/scripts/tests/test_wasm_coreclr_r2r.py index 2ae08bcbf51..5226c8f1c20 100644 --- a/scripts/tests/test_wasm_coreclr_r2r.py +++ b/scripts/tests/test_wasm_coreclr_r2r.py @@ -3,6 +3,7 @@ import shutil import subprocess import sys +import tarfile import xml.etree.ElementTree as ET from argparse import Namespace from pathlib import Path @@ -15,7 +16,12 @@ import micro_benchmarks import dotnet -from build_runtime_payload import build_wasm_coreclr_payload +from build_runtime_payload import ( + WASM_CROSSGEN2_TASKS_FILES, + WASM_CROSSGEN2_TASKS_PAYLOAD_DIR, + _stage_wasm_crossgen2_tasks, + build_wasm_coreclr_payload, +) from run_performance_job import ( get_pre_commands, get_run_configurations, @@ -46,6 +52,70 @@ def test_ready_to_run_configures_msbuild_environment(monkeypatch): assert os.environ["PERFLAB_WASM_READY_TO_RUN"] == "true" +@pytest.mark.parametrize( + "extra_args", + [ + [], + ["--wasm"], + ["--wasm", "--wasm-runtime-flavor", "Mono"], + ], +) +def test_composite_ready_to_run_requires_coreclr_wasm(extra_args): + with pytest.raises(SystemExit): + micro_benchmarks.__process_arguments([ + "--frameworks", "net11.0", + "--wasm-ready-to-run-composite", + *extra_args, + ]) + + +@pytest.mark.parametrize( + ("flags", "expected_ready_to_run", "expected_composite"), + [ + ([], False, False), + (["--wasm-ready-to-run"], True, False), + (["--wasm-ready-to-run-composite"], True, True), + (["--wasm-ready-to-run", "--wasm-ready-to-run-composite"], True, True), + ], +) +def test_ready_to_run_mode_parsing(flags, expected_ready_to_run, expected_composite): + args = micro_benchmarks.__process_arguments([ + "--frameworks", "net11.0", + "--wasm", + "--wasm-runtime-flavor", "CoreCLR", + *flags, + ]) + + assert micro_benchmarks.is_wasm_ready_to_run(args) == expected_ready_to_run + assert micro_benchmarks.is_wasm_ready_to_run_composite(args) == expected_composite + + +@pytest.mark.parametrize( + ("ready_to_run", "composite", "expected_ready_to_run", "expected_composite"), + [ + (False, False, "false", "false"), + (True, False, "true", "false"), + (False, True, "true", "true"), + ], +) +def test_ready_to_run_mode_configures_msbuild_environment( + monkeypatch, ready_to_run, composite, expected_ready_to_run, expected_composite): + # A stale parent value must not leak into a different mode. + monkeypatch.setenv("PERFLAB_WASM_READY_TO_RUN", "true") + monkeypatch.setenv("PERFLAB_WASM_READY_TO_RUN_COMPOSITE", "true") + args = Namespace( + wasm=True, + wasm_runtime_flavor="CoreCLR", + wasm_ready_to_run=ready_to_run, + wasm_ready_to_run_composite=composite, + ) + + micro_benchmarks.configure_wasm_ready_to_run(args) + + assert os.environ["PERFLAB_WASM_READY_TO_RUN"] == expected_ready_to_run + assert os.environ["PERFLAB_WASM_READY_TO_RUN_COMPOSITE"] == expected_composite + + def test_ready_to_run_argument_is_forwarded_to_helix_work_item(): command = get_work_item_command( os_group="linux", @@ -61,6 +131,44 @@ def test_ready_to_run_argument_is_forwarded_to_helix_work_item(): assert "--wasm-runtime-flavor" in command assert "--wasm-ready-to-run" in command + assert "--wasm-ready-to-run-composite" not in command + + +def test_composite_ready_to_run_argument_is_forwarded_to_helix_work_item(): + command = get_work_item_command( + os_group="linux", + target_csproj="src/benchmarks/micro/MicroBenchmarks.csproj", + architecture="x64", + perf_lab_framework="net11.0", + internal=True, + wasm=True, + bdn_artifacts_dir="/tmp/artifacts", + wasm_coreclr=True, + wasm_workload_source="https://example.test/cohort/v3/index.json", + wasm_ready_to_run_composite=True, + ) + + assert "--wasm-runtime-flavor" in command + assert "--wasm-ready-to-run-composite" in command + assert "--wasm-ready-to-run" not in command + source_index = command.index("--wasm-workload-source") + assert command[source_index + 1] == "https://example.test/cohort/v3/index.json" + + +def test_composite_ready_to_run_is_ignored_without_coreclr_wasm(): + command = get_work_item_command( + os_group="linux", + target_csproj="src/benchmarks/micro/MicroBenchmarks.csproj", + architecture="x64", + perf_lab_framework="net11.0", + internal=True, + wasm=True, + bdn_artifacts_dir="/tmp/artifacts", + wasm_coreclr=False, + wasm_ready_to_run_composite=True, + ) + + assert "--wasm-ready-to-run-composite" not in command def test_workload_source_is_forwarded_to_helix_work_item(): @@ -120,6 +228,96 @@ def test_ready_to_run_has_distinct_result_configuration(): assert configurations["R2RType"] == "r2r" +def test_composite_ready_to_run_has_distinct_result_configuration(): + configurations = get_run_configurations( + run_kind="micro", + runtime_type="wasm_coreclr", + codegen_type="wasm", + r2r_run_type="r2r_composite", + runtime_flavor="coreclr", + javascript_engine="v8", + ) + + assert configurations["CompilationMode"] == "wasm" + assert configurations["RuntimeType"] == "coreclr" + assert configurations["R2RType"] == "r2r_composite" + + +def _wasm_targets(): + targets_path = scripts_dir.parent / "src" / "benchmarks" / "micro" / "MicroBenchmarks.Wasm.targets" + return ET.parse(targets_path).getroot() + + +def test_ready_to_run_properties_select_composite_from_environment(): + root = _wasm_targets() + mode = [ + element for group in root.findall("PropertyGroup") + for element in group.findall("_PerformanceWasmReadyToRunComposite") + ] + assert [(e.text, e.attrib.get("Condition")) for e in mode] == [ + ("false", None), + ("true", "'$(PERFLAB_WASM_READY_TO_RUN_COMPOSITE)' == 'true'"), + ] + + r2r_group = next( + group for group in root.findall("PropertyGroup") + if group.find("PublishReadyToRun") is not None + ) + assert "'$(PERFLAB_WASM_READY_TO_RUN)' == 'true'" in r2r_group.attrib["Condition"] + properties = {element.tag: element.text for element in r2r_group} + assert properties == { + "PublishReadyToRun": "true", + "PublishReadyToRunComposite": "$(_PerformanceWasmReadyToRunComposite)", + "PublishReadyToRunContainerFormat": "wasm", + "PublishTrimmed": "true", + "WasmEnableWebcil": "true", + } + + +def test_ready_to_run_configuration_validates_selected_composite_mode(): + target = _wasm_targets().find("./Target[@Name='ValidateWasmReadyToRunConfiguration']") + assert target is not None + assert "PERFLAB_WASM_READY_TO_RUN_COMPOSITE" in target.attrib["Condition"] + + conditions = [error.attrib["Condition"] for error in target.findall("Error")] + assert "'$(PERFLAB_WASM_READY_TO_RUN)' != 'true'" in conditions + assert any( + "'$(PublishReadyToRunComposite)' != '$(_PerformanceWasmReadyToRunComposite)'" in condition + for condition in conditions + ) + + +def test_ready_to_run_output_guard_checks_composite_image(): + target = _wasm_targets().find("./Target[@Name='ValidateWasmReadyToRunOutputs']") + assert target is not None + assert target.attrib["AfterTargets"] == "_CreateR2RImages" + + include = target.find("./ItemGroup/_PerformanceWasmCompositeCompilation").attrib["Include"] + assert "WithMetadataValue('CreateCompositeImage', 'true')" in include + image = target.find("./ItemGroup/_PerformanceWasmCompositeImage").attrib["Include"] + assert "%(OutputR2RImage)" in image + + conditions = " ".join(error.attrib["Condition"] for error in target.findall("Error")) + assert "'@(_ReadyToRunFilesToPublish)' == ''" in conditions + assert "EndsWith('.r2r.wasm')" in conditions + assert "!Exists('$(_PerformanceWasmCompositeImagePath)')" in conditions + assert "@(_ReadyToRunCompositeBuildInput)" in conditions + # Per-assembly mode rejects an unexpected composite plan. + assert "'$(_PerformanceWasmReadyToRunComposite)' != 'true' and '@(_PerformanceWasmCompositeImage)' != ''" in conditions + + +def test_composite_publish_guard_checks_boot_config_core_assembly_asset(): + target = _wasm_targets().find("./Target[@Name='ValidateWasmReadyToRunCompositePublishAssets']") + assert target is not None + assert target.attrib["AfterTargets"] == "ProcessPublishFilesForWasm" + assert "'$(_PerformanceWasmReadyToRunComposite)' == 'true'" in target.attrib["Condition"] + + include = target.find("./ItemGroup/_PerformanceWasmCompositePublishAsset").attrib["Include"] + assert include == ( + "@(_WasmCompositePublishStaticWebAsset->" + "WithMetadataValue('AssetTraitValue', 'readyToRunComposite'))") + + def test_ready_to_run_validates_resolved_runtime_pack_items(): targets_path = scripts_dir.parent / "src" / "benchmarks" / "micro" / "MicroBenchmarks.Wasm.targets" target = ET.parse(targets_path).find("./Target[@Name='ValidateWasmReadyToRunConfiguration']") @@ -225,6 +423,193 @@ def test_coreclr_payload_detects_local_toolchain_package_version(tmp_path): ).is_file() +def _write_coreclr_artifact(tmp_path, crossgen2_tasks_files=()): + artifact = tmp_path / "artifact" / "staging" + shared_framework = artifact / "dotnet-none" / "shared" / "Microsoft.NETCore.App" / "11.0.0-ci" + built_nugets = artifact / "built-nugets" + shared_framework.mkdir(parents=True) + built_nugets.mkdir(parents=True) + (built_nugets / "Microsoft.NET.Sdk.WebAssembly.Pack.11.0.0-ci.nupkg").touch() + (built_nugets / "Microsoft.NETCore.App.Crossgen2.linux-x64.11.0.0-ci.nupkg").touch() + (built_nugets / "Microsoft.NET.ILLink.Tasks.11.0.0-ci.nupkg").touch() + if crossgen2_tasks_files: + crossgen2_tasks = artifact / "Crossgen2Tasks" + crossgen2_tasks.mkdir() + for name in crossgen2_tasks_files: + (crossgen2_tasks / name).write_text(name) + return artifact.parent + + +def test_coreclr_payload_stages_crossgen2_tasks_shim(tmp_path): + artifact = _write_coreclr_artifact( + tmp_path, (*WASM_CROSSGEN2_TASKS_FILES, "Crossgen2Tasks.deps.json")) + payload = tmp_path / "payload" + + build_wasm_coreclr_payload(str(artifact), str(payload)) + + staged = payload / WASM_CROSSGEN2_TASKS_PAYLOAD_DIR + for name in (*WASM_CROSSGEN2_TASKS_FILES, "Crossgen2Tasks.deps.json"): + assert (staged / name).read_text() == name + + +def test_coreclr_payload_without_crossgen2_tasks_shim(tmp_path): + artifact = _write_coreclr_artifact(tmp_path) + payload = tmp_path / "payload" + + build_wasm_coreclr_payload(str(artifact), str(payload)) + + assert not (payload / WASM_CROSSGEN2_TASKS_PAYLOAD_DIR).exists() + + +@pytest.mark.parametrize("files", [(), ("Crossgen2Tasks.deps.json",)]) +def test_coreclr_payload_rejects_present_shim_without_required_files(tmp_path, files): + artifact = _write_coreclr_artifact(tmp_path) + shim = artifact / "staging" / "Crossgen2Tasks" + shim.mkdir() + for name in files: + (shim / name).touch() + + with pytest.raises(ValueError, match="Incomplete Crossgen2Tasks shim"): + build_wasm_coreclr_payload(str(artifact), str(tmp_path / "payload")) + + +def _archive_coreclr_artifact(artifact_root, tmp_path): + archive = tmp_path / "BrowserWasmCoreCLR.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + tar.add(artifact_root / "staging", arcname="staging") + return archive + + +def test_coreclr_archive_payload_without_crossgen2_tasks_shim(tmp_path): + archive = _archive_coreclr_artifact(_write_coreclr_artifact(tmp_path), tmp_path) + payload = tmp_path / "payload" + + assert not _stage_wasm_crossgen2_tasks(str(archive), str(payload)) + assert not (payload / WASM_CROSSGEN2_TASKS_PAYLOAD_DIR).exists() + + +def test_coreclr_archive_payload_stages_crossgen2_tasks_shim(tmp_path): + archive = _archive_coreclr_artifact( + _write_coreclr_artifact(tmp_path, WASM_CROSSGEN2_TASKS_FILES), tmp_path) + payload = tmp_path / "payload" + + assert _stage_wasm_crossgen2_tasks(str(archive), str(payload)) + for name in WASM_CROSSGEN2_TASKS_FILES: + assert (payload / WASM_CROSSGEN2_TASKS_PAYLOAD_DIR / name).is_file() + + +def test_coreclr_archive_payload_rejects_incomplete_crossgen2_tasks_shim(tmp_path): + archive = _archive_coreclr_artifact( + _write_coreclr_artifact(tmp_path, ("Crossgen2Tasks.deps.json",)), tmp_path) + + with pytest.raises(ValueError, match="Incomplete Crossgen2Tasks shim"): + _stage_wasm_crossgen2_tasks(str(archive), str(tmp_path / "payload")) + + +def test_coreclr_payload_rejects_incomplete_crossgen2_tasks_shim(tmp_path): + artifact = _write_coreclr_artifact(tmp_path, ("Crossgen2Tasks.dll",)) + + with pytest.raises(ValueError, match="Microsoft.NET.CrossGen.props"): + build_wasm_coreclr_payload(str(artifact), str(tmp_path / "payload")) + + +def test_coreclr_pre_commands_export_crossgen2_tasks_dir(): + commands = get_pre_commands( + os_group="linux", + os_distro="ubuntu", + internal=False, + runtime_type="wasm_coreclr", + codegen_type="wasm", + build_config="Release", + v8_version="15.1.206", + wasm_local_package_version="11.0.0-ci", + wasm_crossgen2_tasks_dir="$HELIX_CORRELATION_PAYLOAD/crossgen2-tasks", + ) + + assert any( + "export PERFLAB_WASM_CROSSGEN2_TASKS_DIR=$HELIX_CORRELATION_PAYLOAD/crossgen2-tasks" in command + for command in commands + ) + + +def test_coreclr_pre_commands_omit_crossgen2_tasks_dir_by_default(): + commands = get_pre_commands( + os_group="linux", + os_distro="ubuntu", + internal=False, + runtime_type="wasm_coreclr", + codegen_type="wasm", + build_config="Release", + v8_version="15.1.206", + wasm_local_package_version="11.0.0-ci", + ) + + assert not any("PERFLAB_WASM_CROSSGEN2_TASKS_DIR" in command for command in commands) + + +def _crossgen2_tasks_dir(tmp_path, files=WASM_CROSSGEN2_TASKS_FILES): + tasks_dir = tmp_path / "crossgen2-tasks" + tasks_dir.mkdir() + for name in files: + (tasks_dir / name).touch() + return tasks_dir + + +def _ready_to_run_args(composite): + return Namespace( + wasm=True, + wasm_runtime_flavor="CoreCLR", + wasm_ready_to_run=not composite, + wasm_ready_to_run_composite=composite, + ) + + +def test_composite_ready_to_run_activates_crossgen2_tasks_shim(tmp_path, monkeypatch): + tasks_dir = _crossgen2_tasks_dir(tmp_path) + monkeypatch.setenv("PERFLAB_WASM_CROSSGEN2_TASKS_DIR", str(tasks_dir)) + monkeypatch.setenv("Crossgen2SdkOverridePropsPath", "/stale/Microsoft.NET.CrossGen.props") + monkeypatch.setenv("Crossgen2SdkOverrideTargetsPath", "/stale/Microsoft.NET.CrossGen.targets") + + micro_benchmarks.configure_wasm_ready_to_run(_ready_to_run_args(composite=True)) + + assert os.environ["Crossgen2SdkOverridePropsPath"] == str(tasks_dir / "Microsoft.NET.CrossGen.props") + assert os.environ["Crossgen2SdkOverrideTargetsPath"] == str(tasks_dir / "Microsoft.NET.CrossGen.targets") + + +def test_per_assembly_ready_to_run_keeps_sdk_ready_to_run_tasks(tmp_path, monkeypatch): + monkeypatch.setenv("PERFLAB_WASM_CROSSGEN2_TASKS_DIR", str(_crossgen2_tasks_dir(tmp_path))) + # Inherited shim paths must not leak into a per-assembly run. + monkeypatch.setenv("Crossgen2SdkOverridePropsPath", "/stale/Microsoft.NET.CrossGen.props") + monkeypatch.setenv("Crossgen2SdkOverrideTargetsPath", "/stale/Microsoft.NET.CrossGen.targets") + + micro_benchmarks.configure_wasm_ready_to_run(_ready_to_run_args(composite=False)) + + assert "Crossgen2SdkOverridePropsPath" not in os.environ + assert "Crossgen2SdkOverrideTargetsPath" not in os.environ + + +def test_composite_ready_to_run_without_shim_keeps_sdk_ready_to_run_tasks(monkeypatch): + monkeypatch.delenv("PERFLAB_WASM_CROSSGEN2_TASKS_DIR", raising=False) + # A stale inherited shim must not be used when this run has none. + monkeypatch.setenv("Crossgen2SdkOverridePropsPath", "/stale/Microsoft.NET.CrossGen.props") + monkeypatch.setenv("Crossgen2SdkOverrideTargetsPath", "/stale/Microsoft.NET.CrossGen.targets") + + micro_benchmarks.configure_wasm_ready_to_run(_ready_to_run_args(composite=True)) + + assert "Crossgen2SdkOverridePropsPath" not in os.environ + assert "Crossgen2SdkOverrideTargetsPath" not in os.environ + + +def test_composite_ready_to_run_rejects_incomplete_shim(tmp_path, monkeypatch): + tasks_dir = _crossgen2_tasks_dir(tmp_path, ("Crossgen2Tasks.dll", "Microsoft.NET.CrossGen.props")) + monkeypatch.setenv("PERFLAB_WASM_CROSSGEN2_TASKS_DIR", str(tasks_dir)) + monkeypatch.delenv("Crossgen2SdkOverridePropsPath", raising=False) + monkeypatch.delenv("Crossgen2SdkOverrideTargetsPath", raising=False) + + with pytest.raises(FileNotFoundError, match="Microsoft.NET.CrossGen.targets"): + micro_benchmarks.configure_wasm_ready_to_run(_ready_to_run_args(composite=True)) + + def test_coreclr_payload_aliases_sdk_framework_during_major_version_rollover(tmp_path): artifact = tmp_path / "artifact" / "staging" dotnet = artifact / "dotnet-none" @@ -392,11 +777,43 @@ def test_pipeline_scopes_workload_source_to_coreclr_r2r(): condition = ( "and(ne(parameters.wasmWorkloadSource, ''), " "eq(parameters.runtimeType, 'wasm_coreclr'), " - "eq(parameters.r2rRunType, 'r2r'))" + "in(parameters.r2rRunType, 'r2r', 'r2r_composite'))" ) assert condition in template +def test_pipeline_defines_coreclr_composite_r2r_lane(): + jobs = ( + scripts_dir.parent / "eng" / "pipelines" / "runtime-wasm-perf-jobs.yml" + ).read_text(encoding="utf-8") + release_exclusion = ( + " - ${{ if not(startswith(variables['Build.SourceBranch'], " + "'refs/heads/release')) }}:\n" + ) + + def lane(identifier): + blocks = [ + block for block in jobs.split(release_exclusion)[1:] + if f"additionalJobIdentifier: {identifier}\n" in block + ] + assert len(blocks) == 1 + # Stop at the next job's leading comment. + return blocks[0].split("\n\n")[0] + + def without_lane_identity(block): + return [ + line for line in block.splitlines() + if not line.strip().startswith(("r2rRunType:", "additionalJobIdentifier:")) + ] + + per_assembly = lane("coreclr_r2r_v8") + composite = lane("coreclr_r2r_composite_v8") + + assert "r2rRunType: 'r2r'\n" in per_assembly + assert "r2rRunType: 'r2r_composite'\n" in composite + assert without_lane_identity(composite) == without_lane_identity(per_assembly) + + @pytest.mark.parametrize("source", [None, "", " "]) def test_empty_workload_source_is_not_enabled(source): assert normalize_wasm_workload_source(source) is None diff --git a/src/benchmarks/micro/MicroBenchmarks.Wasm.targets b/src/benchmarks/micro/MicroBenchmarks.Wasm.targets index e1a4340fe4c..496712b19a4 100644 --- a/src/benchmarks/micro/MicroBenchmarks.Wasm.targets +++ b/src/benchmarks/micro/MicroBenchmarks.Wasm.targets @@ -15,9 +15,17 @@ CoreCLR + + + <_PerformanceWasmReadyToRunComposite>false + <_PerformanceWasmReadyToRunComposite Condition="'$(PERFLAB_WASM_READY_TO_RUN_COMPOSITE)' == 'true'">true + + true - false + $(_PerformanceWasmReadyToRunComposite) wasm true true @@ -26,16 +34,18 @@ + Condition="'$(PERFLAB_WASM_READY_TO_RUN)' == 'true' or '$(PERFLAB_WASM_READY_TO_RUN_COMPOSITE)' == 'true'"> <_PerformanceWasmResolvedRuntimePack Include="@(ResolvedRuntimePack)" Condition="'%(ResolvedRuntimePack.FrameworkName)' == 'Microsoft.NETCore.App'" /> <_PerformanceWasmResolvedFrameworkReference Include="@(ResolvedFrameworkReference)" Condition="'%(ResolvedFrameworkReference.Identity)' == 'Microsoft.NETCore.App'" /> + - @@ -46,8 +56,42 @@ + + <_PerformanceWasmCompositeCompilation Include="@(_ReadyToRunCompileList->WithMetadataValue('CreateCompositeImage', 'true'))" /> + <_PerformanceWasmCompositeImage Include="@(_PerformanceWasmCompositeCompilation->'%(OutputR2RImage)')" /> + + + <_PerformanceWasmCompositeImagePath>@(_PerformanceWasmCompositeImage) + + + + + + + + + + + + + <_PerformanceWasmCompositePublishAsset Include="@(_WasmCompositePublishStaticWebAsset->WithMetadataValue('AssetTraitValue', 'readyToRunComposite'))" /> + + +