Build all platforms in parallel in release pipeline

Herebyfile.mjs

@@ -87,6 +87,7 @@ const { values: rawOptions } = parseArgs({
         forRelease: { type: "boolean" },
         respectGoEnv: { type: "boolean" },
         vscodeTypescriptRelease: { type: "boolean" },
+        platform: { type: "string" },
 
         race: { type: "boolean", default: parseEnvBoolean("RACE") },
         noembed: { type: "boolean", default: parseEnvBoolean("NOEMBED") },
@@ -2216,46 +2217,79 @@ async function runBuildNativePreviewPackages() {
     await rimraf(builtNpm);
 
     const platforms = getPlatforms();
+    await buildMainNativePreviewPackage(platforms);
 
-    const inputDir = "./packages/typescript";
-
-    const inputPackageJson = JSON.parse(fs.readFileSync(path.join(inputDir, "package.json"), "utf8"));
-    inputPackageJson.version = getVersion();
-    delete inputPackageJson.private;
-    inputPackageJson.files = [...new Set([...(inputPackageJson.files ?? []), "NOTICE.txt"])];
-    if (publishAsTypescript) {
-        inputPackageJson.bin = {
-            tsc: "./bin/tsc",
-        };
-        inputPackageJson.description = "TypeScript is a language for application scale JavaScript development";
-        inputPackageJson.homepage = "https://www.typescriptlang.org/";
-        inputPackageJson.keywords = [
-            "TypeScript",
-            "Microsoft",
-            "compiler",
-            "language",
-            "javascript",
-        ];
-        inputPackageJson.bugs = {
-            url: "https://github.com/microsoft/TypeScript/issues",
-        };
-        inputPackageJson.repository = {
-            type: "git",
-            url: "https://github.com/microsoft/TypeScript.git",
-        };
-        delete inputPackageJson.scripts;
-        delete inputPackageJson.devDependencies;
+    if (isCI) {
+        for (const platform of platforms) {
+            await buildNativePreviewPlatform(platform);
+            // Build machines have too little space.
+            // Clear the Go build cache between platforms.
+            await run("go", ["clean", "-cache"]);
+        }
     }
-    stripSourceConditions(inputPackageJson);
-
-    if (fs.existsSync(".git")) {
-        const { stdout: gitHead } = await runOutput("git", ["rev-parse", "HEAD"]);
-        inputPackageJson.gitHead = gitHead.trim();
+    else {
+        await runWithConcurrencyLimit(platforms.map(platform => () => buildNativePreviewPlatform(platform)), os.availableParallelism());
     }
-    inputPackageJson.publishConfig = {
-        access: "public",
-        tag: getPublishTag(),
-    };
+}
+
+export const buildMainNativePreviewPackageTask = task({
+    name: "typescript:build-main",
+    hiddenFromTaskList: true,
+    run: async () => {
+        if (usePublishedPlatformPackagesForVsix) {
+            checkPublishedPlatformPackagesForVsix();
+            await rimraf(builtNpm);
+            console.log("Skipping npm package build; VSIX packaging will use published platform packages.");
+            return;
+        }
+
+        await rimraf(builtNpm);
+        await buildMainNativePreviewPackage(getPlatforms());
+    },
+});
+
+export const buildNativePreviewPlatformTask = task({
+    name: "typescript:build-platform",
+    hiddenFromTaskList: true,
+    run: async () => {
+        if (usePublishedPlatformPackagesForVsix) {
+            throw new Error("Platform packages are not built when VSIX packaging uses published platform packages.");
+        }
+        if (!options.platform) {
+            throw new Error("typescript:build-platform requires --platform.");
+        }
+
+        const platform = getPlatforms().find(({ nodeOs, nodeArch }) => `${nodeOs}-${nodeArch}` === options.platform);
+        if (!platform) {
+            throw new Error(`Unknown release platform: ${options.platform}`);
+        }
+
+        await rimraf(platform.npmDir);
+        await buildNativePreviewPlatform(platform);
+    },
+});
+
+export const buildNativePreviewPlatformMatrix = task({
+    name: "typescript:build-platform-matrix",
+    hiddenFromTaskList: true,
+    run: () => {
+        const matrix = Object.fromEntries(
+            getPlatforms().map(({ nodeOs, nodeArch, npmDirName }) => {
+                const platform = `${nodeOs}-${nodeArch}`;
+                return [platform.replaceAll("-", "_"), { platform, npmDirName }];
+            }),
+        );
+        console.log(`##vso[task.setvariable variable=matrix;isOutput=true]${JSON.stringify(matrix)}`);
+        console.log(`##vso[task.setvariable variable=packageBaseName;isOutput=true]${path.basename(mainNativePreviewPackage.npmDir)}`);
+    },
+});
+
+/**
+ * @param {ReturnType<typeof getPlatforms>} platforms
+ */
+async function buildMainNativePreviewPackage(platforms) {
+    const inputDir = "./packages/typescript";
+    const inputPackageJson = await createNativePreviewInputPackageJson();
 
     const mainPackage = {
         ...inputPackageJson,
@@ -2307,65 +2341,98 @@ async function runBuildNativePreviewPackages() {
     if (importErrors.length) {
         throw new Error(`Found external imports in .d.ts files:\n${importErrors.map(e => "  " + e).join("\n")}`);
     }
+}
 
-    const extraFlags = options.respectGoEnv
-        ? []
-        : getReleaseBuildFlags(options.setPrerelease || nativePreviewReleaseVersion ? getVersion() : undefined);
-
-    const platformBuilders = platforms.map(({ npmDir, npmPackageName, nodeOs, nodeArch, goos, goarch }) => async () => {
-        const packageJson = {
-            ...inputPackageJson,
-            bin: undefined,
-            files: ["lib", "NOTICE.txt"],
-            imports: undefined,
-            dependencies: undefined,
-            name: npmPackageName,
-            os: [nodeOs],
-            cpu: [nodeArch],
-            exports: {
-                "./package.json": "./package.json",
-            },
-        };
+/**
+ * @param {ReturnType<typeof getPlatforms>[number]} platform
+ */
+async function buildNativePreviewPlatform({ npmDir, npmPackageName, nodeOs, nodeArch, goos, goarch }) {
+    const inputPackageJson = await createNativePreviewInputPackageJson();
 
-        const out = path.join(npmDir, "lib");
-        await fs.promises.mkdir(out, { recursive: true });
-        await fs.promises.writeFile(path.join(npmDir, "package.json"), JSON.stringify(packageJson, undefined, 4));
-        await fs.promises.copyFile("LICENSE.txt", path.join(npmDir, "LICENSE"));
-        await fs.promises.copyFile("NOTICE.txt", path.join(npmDir, "NOTICE.txt"));
+    const packageJson = {
+        ...inputPackageJson,
+        bin: undefined,
+        files: ["lib", "NOTICE.txt"],
+        imports: undefined,
+        dependencies: undefined,
+        name: npmPackageName,
+        os: [nodeOs],
+        cpu: [nodeArch],
+        exports: {
+            "./package.json": "./package.json",
+        },
+    };
 
-        const readme = [
-            `# \`${npmPackageName}\``,
-            "",
-            `This package provides ${nodeOs}-${nodeArch} support for [${mainNativePreviewPackage.npmPackageName}](https://www.npmjs.com/package/${mainNativePreviewPackage.npmPackageName}).`,
-        ];
+    const out = path.join(npmDir, "lib");
+    await fs.promises.mkdir(out, { recursive: true });
+    await fs.promises.writeFile(path.join(npmDir, "package.json"), JSON.stringify(packageJson, undefined, 4));
+    await fs.promises.copyFile("LICENSE.txt", path.join(npmDir, "LICENSE"));
+    await fs.promises.copyFile("NOTICE.txt", path.join(npmDir, "NOTICE.txt"));
+
+    const readme = [
+        `# \`${npmPackageName}\``,
+        "",
+        `This package provides ${nodeOs}-${nodeArch} support for [${mainNativePreviewPackage.npmPackageName}](https://www.npmjs.com/package/${mainNativePreviewPackage.npmPackageName}).`,
+    ];
 
-        await fs.promises.writeFile(path.join(npmDir, "README.md"), readme.join("\n") + "\n");
+    await fs.promises.writeFile(path.join(npmDir, "README.md"), readme.join("\n") + "\n");
 
-        await generateLibs(out);
+    await generateLibs(out);
 
-        const exeName = nativePreviewExeName(nodeOs);
-        await buildTsc({
-            out: publishAsTypescript ? path.join(out, exeName) : out,
-            env: {
-                GOOS: goos,
-                GOARCH: goarch,
-                ...(options.respectGoEnv ? {} : { GOARM: "6", CGO_ENABLED: "0" }),
-            },
-            extraFlags,
-        });
+    const exeName = nativePreviewExeName(nodeOs);
+    await buildTsc({
+        out: publishAsTypescript ? path.join(out, exeName) : out,
+        env: {
+            GOOS: goos,
+            GOARCH: goarch,
+            ...(options.respectGoEnv ? {} : { GOARM: "6", CGO_ENABLED: "0" }),
+        },
+        extraFlags: options.respectGoEnv
+            ? []
+            : getReleaseBuildFlags(options.setPrerelease || nativePreviewReleaseVersion ? getVersion() : undefined),
     });
+}
 
-    if (isCI) {
-        for (const build of platformBuilders) {
-            await build();
-            // Build machines have too little space.
-            // Clear the Go build cache between platforms.
-            await run("go", ["clean", "-cache"]);
-        }
+async function createNativePreviewInputPackageJson() {
+    const inputPackageJson = JSON.parse(fs.readFileSync("./packages/typescript/package.json", "utf8"));
+    inputPackageJson.version = getVersion();
+    delete inputPackageJson.private;
+    inputPackageJson.files = [...new Set([...(inputPackageJson.files ?? []), "NOTICE.txt"])];
+    if (publishAsTypescript) {
+        inputPackageJson.bin = {
+            tsc: "./bin/tsc",
+        };
+        inputPackageJson.description = "TypeScript is a language for application scale JavaScript development";
+        inputPackageJson.homepage = "https://www.typescriptlang.org/";
+        inputPackageJson.keywords = [
+            "TypeScript",
+            "Microsoft",
+            "compiler",
+            "language",
+            "javascript",
+        ];
+        inputPackageJson.bugs = {
+            url: "https://github.com/microsoft/TypeScript/issues",
+        };
+        inputPackageJson.repository = {
+            type: "git",
+            url: "https://github.com/microsoft/TypeScript.git",
+        };
+        delete inputPackageJson.scripts;
+        delete inputPackageJson.devDependencies;
     }
-    else {
-        await runWithConcurrencyLimit(platformBuilders, os.availableParallelism());
+    stripSourceConditions(inputPackageJson);
+
+    if (fs.existsSync(".git")) {
+        const { stdout: gitHead } = await runOutput("git", ["rev-parse", "HEAD"]);
+        inputPackageJson.gitHead = gitHead.trim();
     }
+    inputPackageJson.publishConfig = {
+        access: "public",
+        tag: getPublishTag(),
+    };
+
+    return inputPackageJson;
 }
 
 /**

tools/pipelines/typescript-build.yml

@@ -51,13 +51,68 @@ extends:
       - stage: Build
         displayName: Build and Sign
         jobs:
+          - job: SetupBuildMatrix
+            displayName: Set up platform build matrix
+
+            templateContext:
+              mb:
+                signing:
+                  enabled: false
+
+            steps:
+              - checkout: self
+                clean: true
+                submodules: false
+                fetchDepth: 1
+                fetchTags: false
+              - template: /tools/pipelines/steps/setup-node-npm-ci.yml@self
+
+              - bash: npx hereby typescript:build-platform-matrix --forRelease --setPrerelease dev.$(Build.BuildNumber)
+                displayName: Set build matrix
+                name: buildMatrix
+
+          - job: BuildPackages
+            displayName: Build platform package
+            dependsOn: SetupBuildMatrix
+
+            strategy:
+              matrix: $[ dependencies.SetupBuildMatrix.outputs['buildMatrix.matrix'] ]
+
+            templateContext:
+              mb:
+                signing:
+                  enabled: false
+              outputs:
+                - output: pipelineArtifact
+                  targetPath: $(Build.SourcesDirectory)/built/npm/$(npmDirName)
+                  artifactName: $(npmDirName)
+                  isProduction: false
+                  sbomEnabled: false
+
+            steps:
+              - checkout: self
+                clean: true
+                submodules: false
+                fetchDepth: 1
+                fetchTags: false
+              - template: /tools/pipelines/steps/setup-node-npm-ci.yml@self
+              - template: /tools/pipelines/steps/setup-go.yml@self
+
+              - bash: npx hereby typescript:build-platform --platform $(platform) --forRelease --setPrerelease dev.$(Build.BuildNumber)
+                displayName: Build $(platform)
+
           - job: Build
             displayName: Build and Sign
+            dependsOn:
+              - SetupBuildMatrix
+              - BuildPackages
             timeoutInMinutes: 180
             pool:
               name: AzurePipelines-EO
               image: 1ESPT-Ubuntu22.04
               os: linux
+            variables:
+              packageBaseName: $[ dependencies.SetupBuildMatrix.outputs['buildMatrix.packageBaseName'] ]
 
             templateContext:
               mb:
@@ -136,8 +191,37 @@ extends:
 
               - template: /tools/pipelines/steps/setup-go.yml@self
 
-              - bash: npx hereby typescript:build --forRelease --setPrerelease dev.$(initialBuildNumber)
-                displayName: 'Build packages'
+              - bash: npx hereby typescript:build-main --forRelease --setPrerelease dev.$(initialBuildNumber)
+                displayName: 'Build main package'
+
+              - task: DownloadPipelineArtifact@2
+                displayName: 'Download platform packages'
+                inputs:
+                  buildType: current
+                  targetPath: $(Pipeline.Workspace)/platform-packages
+                  patterns: '$(packageBaseName)-*/**'
+
+              - bash: |
+                  set -euo pipefail
+                  shopt -s nullglob
+                  platformArtifacts=("$(Pipeline.Workspace)"/platform-packages/"$PACKAGE_BASE_NAME"-*)
+                  if (( ${#platformArtifacts[@]} == 0 )); then
+                    echo "No platform package artifacts were downloaded." >&2
+                    exit 1
+                  fi
+                  for artifact in "${platformArtifacts[@]}"; do
+                    packageDir="built/npm/$(basename "$artifact")"
+                    cp -a "$artifact" "$packageDir"
+                    executable="$(find "$packageDir/lib" -maxdepth 1 -type f \( -name tsc -o -name tsc.exe -o -name tsgo -o -name tsgo.exe \) -print -quit)"
+                    if [[ -z "$executable" ]]; then
+                      echo "Missing platform executable in $packageDir." >&2
+                      exit 1
+                    fi
+                    chmod +x "$executable"
+                  done
+                displayName: 'Assemble platform packages'
+                env:
+                  PACKAGE_BASE_NAME: $(packageBaseName)
 
               - bash: npx hereby typescript:test-package --forRelease --setPrerelease dev.$(initialBuildNumber)
                 displayName: 'Test packages'