From 7c73646d39aa12ed6e9613d4174d3fe932d8c8fc Mon Sep 17 00:00:00 2001 From: thobed <10742470+thobed@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:23:50 -0400 Subject: [PATCH 1/3] perf(ci): cut deploy time from ~55min to a few minutes Two deploy steps each walked the whole `$web` container. `Set public URL metadata on blobs` spawned one Azure CLI process per blob, 20 at a time; at ~0.66s of cold start each and ~86k blobs that step alone took 47 of the 70 minutes a push to main spent in CI, and it re-stamped every blob regardless of what changed. `Set MIME types for all file types` then made 19 more passes, one `az storage blob update-batch` per file extension. Fold both into a single listing pass in scripts/set-blob-metadata.mjs. `listBlobsFlat({ includeMetadata: true })` returns each blob's metadata and properties inline, so deciding what needs writing costs no extra round trips and only missing or stale blobs get a PUT, over a bounded pool rather than a process-per-blob fan-out. Pass --force to rewrite unconditionally, or --dry-run to list and plan without writing. Blob names are percent-encoded per path segment before going into the metadata value. The old loop interpolated them raw, which is not safe: the site ships names carrying a Cyrillic homoglyph and a curly quote, and both throw at socket-write time because `x-ms-meta-*` headers are latin-1 on the wire. Those writes failed silently before, since each `az` call was backgrounded and its exit code never checked. Unencoded spaces also produced invalid URLs. Failures are now reported per blob and fail the step. Content types are set with the other content settings carried through, since setHTTPHeaders replaces every header it accepts rather than merging. Also: - Merge the build and deploy jobs. Handing the 1.5GB build/ directory between two runners cost ~4.5min in artifact upload plus download for no benefit now that both halves run in one place. Deploy steps are gated on the event not being a pull_request, so PR runs build and stop as before. The build-output artifact upload is kept, now PR-only. - Drop the Azure CLI install and every use of sudo. The deploy now runs on the self-hosted gh-runner-large, where sudo cannot be assumed. azcopy is unpacked into $RUNNER_TEMP and added to $GITHUB_PATH, and the SAS token azcopy uploads with is minted by scripts/mint-container-sas.mjs using the storage SDK already in node_modules. - Fix the compiler cache key. It was keyed on package-lock.json alone, and actions/cache only saves on a key miss, so after the first run it hit that key forever and never saved a refreshed cache. Appending github.run_id makes every run miss-then-save while restore-keys pull the newest prior cache. - Drop the .docusaurus cache. Its key hashed docs/**, so any content change missed it, and the restore-keys fallback only ever restored a stale directory that Docusaurus regenerates anyway. - Enable ssgWorkerThreads. Measured on a scoped pingcastle build (1595 pages, two rounds each): 59.7s -> 49.8s mean, with peak heap dropping from 680MB to 313MB since rendering moves off the main thread. --- .github/workflows/build-and-deploy.yml | 252 ++++++---------- docusaurus.config.js | 4 +- package-lock.json | 381 +++++++++++++++++++++++++ package.json | 3 +- scripts/mint-container-sas.mjs | 51 ++++ scripts/set-blob-metadata.mjs | 261 +++++++++++++++++ scripts/test-set-blob-metadata.mjs | 175 ++++++++++++ 7 files changed, 965 insertions(+), 162 deletions(-) create mode 100644 scripts/mint-container-sas.mjs create mode 100644 scripts/set-blob-metadata.mjs create mode 100644 scripts/test-set-blob-metadata.mjs diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index 4fc5345c8b..02086073ad 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -40,7 +40,10 @@ jobs: echo "environment=development" >> $GITHUB_OUTPUT fi - build: + # Build and deploy share one job so the 1.5 GB build/ directory never makes a + # round trip through the artifact store. Handing it between two jobs cost about + # 4.5 minutes (1.5 min upload + 3 min download) purely in transfer. + build-and-deploy: runs-on: gh-runner-large needs: determine-environment environment: ${{ needs.determine-environment.outputs.environment }} @@ -69,185 +72,114 @@ jobs: restore-keys: | v2-${{ runner.os }}-node- - - name: Cache Docusaurus build - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - id: cache-build - with: - path: | - .docusaurus - key: v2-${{ runner.os }}-docusaurus-${{ hashFiles('src/**', 'docs/**', 'blog/**', 'docusaurus.config.js', 'sidebars.js') }} - restore-keys: | - v2-${{ runner.os }}-docusaurus- - - - name: Cache webpack + # Rspack's persistent cache (future.faster.rspackPersistentCache) and the + # MDX cross-compiler cache both live here, and this is the cache that + # actually shortens a rebuild. The key carries github.run_id so every run + # misses on the exact key and therefore *saves* a refreshed cache, while + # restore-keys pull the newest previous one. Keying on package-lock.json + # alone (the previous behaviour) meant the first run saved a cache that + # then hit forever and never picked up newly compiled modules. + # + # The former `.docusaurus` cache was dropped: its key hashed docs/**, so + # any content change missed it, and the restore-keys fallback just + # restored a stale directory that Docusaurus regenerates anyway. + - name: Cache Rspack/MDX compiler output uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: node_modules/.cache - key: v2-${{ runner.os }}-webpack-${{ hashFiles('**/package-lock.json') }} + key: v3-${{ runner.os }}-rspack-${{ hashFiles('**/package-lock.json') }}-${{ github.run_id }} restore-keys: | - v2-${{ runner.os }}-webpack- + v3-${{ runner.os }}-rspack-${{ hashFiles('**/package-lock.json') }}- + v3-${{ runner.os }}-rspack- - name: Install dependencies and build site run: | npm ci - - if [[ "${{ steps.cache-build.outputs.cache-hit }}" == "true" ]]; then - echo "Build cache found, checking if rebuild needed..." - else - echo "No build cache found, performing full build..." - fi - npm run ci env: NODE_OPTIONS: "--max-old-space-size=16384" NODE_ENV: ${{ needs.determine-environment.outputs.environment }} + # PR runs stop before the deploy steps below, so the built site is only + # reachable as an artifact. Kept PR-only: on push the build is deployed, + # and uploading 1.5GB there cost ~4.5min for nothing. - name: Upload artifact for deployment + if: github.event_name == 'pull_request' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: build-output path: build/ - deploy: - runs-on: ubuntu-latest - if: github.event_name != 'pull_request' - needs: [build, determine-environment] - environment: ${{ needs.determine-environment.outputs.environment }} - steps: - - name: Download build artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: build-output - path: build/ - - - name: Install azcopy - run: | - wget -O azcopy.tar.gz https://aka.ms/downloadazcopy-v10-linux - tar -xf azcopy.tar.gz --strip-components=1 - sudo mv azcopy /usr/local/bin/ - azcopy --version - - - name: Install Azure CLI - run: | - if ! command -v az &> /dev/null; then - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - fi - az version - - - name: Upload to Azure Blob Storage with AzCopy - run: | - echo "Deploying to ${{ needs.determine-environment.outputs.environment }} environment" - echo "Starting sync of changed files..." - - # Create SAS token for azcopy (using account key) - end_date=$(date -u -d "2 hours" '+%Y-%m-%dT%H:%MZ') - sas_token=$(az storage container generate-sas \ - --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} \ - --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} \ - --name '$web' \ - --permissions dlrw \ - --expiry $end_date \ - --output tsv) - - azcopy sync "./build/" \ - "https://${{ secrets.STORAGE_ACCOUNT_NAME }}.blob.core.windows.net/\$web?$sas_token" \ - --delete-destination=true \ - --log-level=INFO \ - --cap-mbps=0 \ - --block-size-mb=4 - - echo "Sync completed!" - - - name: Set MIME types for all file types - run: | - echo "Setting MIME types for all file types..." - - # Web files - echo "Setting MIME types for web files..." - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.css" --content-type "text/css" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.js" --content-type "application/javascript" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.mjs" --content-type "application/javascript" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.json" --content-type "application/json" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.html" --content-type "text/html" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.htm" --content-type "text/html" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.xml" --content-type "application/xml" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.txt" --content-type "text/plain" --no-progress || true - - # Images - echo "Setting MIME types for images..." - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.png" --content-type "image/png" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.jpg" --content-type "image/jpeg" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.jpeg" --content-type "image/jpeg" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.gif" --content-type "image/gif" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.webp" --content-type "image/webp" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.svg" --content-type "image/svg+xml" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.ico" --content-type "image/x-icon" --no-progress || true - - # Fonts - echo "Setting MIME types for fonts..." - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.woff" --content-type "font/woff" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.woff2" --content-type "font/woff2" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.ttf" --content-type "font/ttf" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.otf" --content-type "font/otf" --no-progress || true - - echo "All MIME types set successfully!" - - - name: Set public URL metadata on blobs - env: - APP_EXTERNAL_URL: ${{ secrets.APP_EXTERNAL_URL || vars.APP_EXTERNAL_URL }} - run: | - if [[ -z "$APP_EXTERNAL_URL" ]]; then - echo "APP_EXTERNAL_URL not configured, skipping public URL metadata" - exit 0 - fi - PUBLIC_URL="${APP_EXTERNAL_URL%/}" - - echo "Setting public_url metadata on blobs (base: $PUBLIC_URL)..." - - ACCOUNT="${{ secrets.STORAGE_ACCOUNT_NAME }}" - KEY="${{ secrets.STORAGE_ACCOUNT_KEY }}" - - blob_count=0 - az storage blob list \ - --account-name "$ACCOUNT" \ - --account-key "$KEY" \ - --container-name '$web' \ - --query "[].name" -o tsv | \ - while IFS= read -r blob; do - az storage blob metadata update \ - --account-name "$ACCOUNT" \ - --account-key "$KEY" \ - --container-name '$web' \ - --name "$blob" \ - --metadata "public_url=${PUBLIC_URL}/${blob}" \ - --output none & - - blob_count=$((blob_count + 1)) - # Run up to 20 concurrent updates, then wait for the batch - if (( blob_count % 20 == 0 )); then - wait - echo " Processed $blob_count blobs..." + # Installed into the job's temp dir and put on PATH, so this needs no + # sudo. The self-hosted runner's sudo availability is not something the + # workflow should depend on. + - name: Install azcopy + if: github.event_name != 'pull_request' + run: | + mkdir -p "$RUNNER_TEMP/azcopy" + curl -fsSL -o "$RUNNER_TEMP/azcopy/azcopy.tar.gz" https://aka.ms/downloadazcopy-v10-linux + tar -xf "$RUNNER_TEMP/azcopy/azcopy.tar.gz" -C "$RUNNER_TEMP/azcopy" --strip-components=1 + echo "$RUNNER_TEMP/azcopy" >> "$GITHUB_PATH" + "$RUNNER_TEMP/azcopy/azcopy" --version + + # The SAS token is minted with the storage SDK already in node_modules, + # so the Azure CLI is no longer installed on the runner at all. + - name: Upload to Azure Blob Storage with AzCopy + if: github.event_name != 'pull_request' + env: + STORAGE_ACCOUNT_NAME: ${{ secrets.STORAGE_ACCOUNT_NAME }} + STORAGE_ACCOUNT_KEY: ${{ secrets.STORAGE_ACCOUNT_KEY }} + run: | + echo "Deploying to ${{ needs.determine-environment.outputs.environment }} environment" + echo "Starting sync of changed files..." + + sas_token=$(node scripts/mint-container-sas.mjs --container '$web' --hours 2 --permissions dlrw) + + azcopy sync "./build/" \ + "https://${STORAGE_ACCOUNT_NAME}.blob.core.windows.net/\$web?${sas_token}" \ + --delete-destination=true \ + --log-level=INFO \ + --cap-mbps=0 \ + --block-size-mb=4 + + echo "Sync completed!" + + # Folds together two steps that each walked the whole container: a per-blob + # `az storage blob metadata update` loop (~0.6s of CLI cold start each, 20 at + # a time, ~47 minutes for the site's ~86k blobs) and 19 `az storage blob + # update-batch` calls, one per extension. The script lists metadata and + # properties inline with the blob listing and writes only what is absent or + # stale. Pass --force to rewrite every blob unconditionally. + - name: Set public URL metadata and content types on blobs + if: github.event_name != 'pull_request' + env: + STORAGE_ACCOUNT_NAME: ${{ secrets.STORAGE_ACCOUNT_NAME }} + STORAGE_ACCOUNT_KEY: ${{ secrets.STORAGE_ACCOUNT_KEY }} + run: | + if [[ -z "$APP_EXTERNAL_URL" ]]; then + echo "APP_EXTERNAL_URL not configured, skipping blob metadata" + exit 0 fi - done - wait - echo "Public URL metadata set on all blobs" + node scripts/set-blob-metadata.mjs --container '$web' - - name: Purge CDN endpoint (if configured) - run: | - if [[ -n "${{ secrets.CDN_ENDPOINT_NAME }}" ]] && [[ -n "${{ secrets.CDN_PROFILE_NAME }}" ]] && [[ -n "${{ secrets.CDN_RESOURCE_GROUP }}" ]]; then - echo "Note: CDN purge requires Azure login. Skipping CDN purge when using storage key authentication." - echo "To use CDN purge, you'll need to use Azure AD authentication or purge CDN manually." - else - echo "CDN configuration not found, skipping CDN purge." - fi + - name: Purge CDN endpoint (if configured) + if: github.event_name != 'pull_request' + run: | + if [[ -n "${{ secrets.CDN_ENDPOINT_NAME }}" ]] && [[ -n "${{ secrets.CDN_PROFILE_NAME }}" ]] && [[ -n "${{ secrets.CDN_RESOURCE_GROUP }}" ]]; then + echo "Note: CDN purge requires Azure login. Skipping CDN purge when using storage key authentication." + echo "To use CDN purge, you'll need to use Azure AD authentication or purge CDN manually." + else + echo "CDN configuration not found, skipping CDN purge." + fi - - name: Display deployment URL - run: | - echo "Deployment complete!" - echo "Environment: ${{ needs.determine-environment.outputs.environment }}" - echo "URL: https://${{ secrets.STORAGE_ACCOUNT_NAME }}.z13.web.core.windows.net" - if [[ -n "${{ secrets.CUSTOM_DOMAIN }}" ]]; then - echo "Custom Domain: ${{ secrets.CUSTOM_DOMAIN }}" - fi - echo "All files deployed with proper MIME types for optimal browser compatibility!" + - name: Display deployment URL + if: github.event_name != 'pull_request' + run: | + echo "Deployment complete!" + echo "Environment: ${{ needs.determine-environment.outputs.environment }}" + echo "URL: https://${{ secrets.STORAGE_ACCOUNT_NAME }}.z13.web.core.windows.net" + if [[ -n "${{ secrets.CUSTOM_DOMAIN }}" ]]; then + echo "Custom Domain: ${{ secrets.CUSTOM_DOMAIN }}" + fi + echo "All files deployed with proper MIME types for optimal browser compatibility!" diff --git a/docusaurus.config.js b/docusaurus.config.js index 5f6cf45ce7..9833ce2c5e 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -127,7 +127,9 @@ const config = { rspackBundler: true, rspackPersistentCache: true, mdxCrossCompilerCache: true, - ssgWorkerThreads: false, + // Renders the static pages across a worker pool instead of one thread. + // Requires future.v4.removeLegacyPostBuildHeadAttribute, set below. + ssgWorkerThreads: true, }, v4: { removeLegacyPostBuildHeadAttribute: true, diff --git a/package-lock.json b/package-lock.json index 26cc40d6f8..e536975db5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "netwrix-docs", "version": "0.1", "dependencies": { + "@azure/storage-blob": "^12.33.0", "@docusaurus/babel": "^3.10.2", "@docusaurus/core": "^3.10.2", "@docusaurus/faster": "^3.10.2", @@ -318,6 +319,208 @@ "@types/json-schema": "^7.0.15" } }, + "node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-auth": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.11.0.tgz", + "integrity": "sha512-IUZydyTUkDnYdstOW9pFOOUQlBjAepK5teihDE3x6yxsPJs/hsAaaYpeGxdxrgtOiJbBKSjKW7MDk7AEhb4LRg==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-client": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.11.1.tgz", + "integrity": "sha512-2QygG2F76ZpMP2eMztiJvAiFMu71M9rDeU7vO/QKg5Css7MgM4frUOslFjhVjRhbGaCNPtz/S8M6y46/fFKVuQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-http-compat": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.5.0.tgz", + "integrity": "sha512-BoSmXPx2er1Ai+wKlDvj29jIQespCNBwEmKyZVHO2kEFsWbGjAjwMCGzug3DJM5/QYIV3vej0S1zcU5bq9fa8w==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "@azure/core-client": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0" + } + }, + "node_modules/@azure/core-lro": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.7.2.tgz", + "integrity": "sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-util": "^1.2.0", + "@azure/logger": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-paging": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.7.0.tgz", + "integrity": "sha512-7GEAoIsaoBr6KELNRb8nypowCqvk8dnCHFCYg4XD4lOQGY2GqjQg5IhkRjyBFRO18CGSMq05PaNqSOE9GQro3g==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.25.0.tgz", + "integrity": "sha512-bMs8ekJLjX8wPV+9IPBges1SLPyuDtE9g5gLDWOpxzKcoOFQnpLGkbcT1tdw3FaAmDS1gnPmMmJ6y/T5B96kIA==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-tracing": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.4.0.tgz", + "integrity": "sha512-eGwxD0AtncrxeBM4tG8R55Pc3rdX1hNW2WibJAgYpCVA6E93mvvVH+LcssoVjOBrSKWS55yEIHsk0X8ctHmfOQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-util": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.14.0.tgz", + "integrity": "sha512-9n2pWK61veAuN0V20t9lOuoV4CFMdyAZ1ygZzvBGk/pBBJRib/PjL9PLXa/aI2CcPpyHfqVsxxqLCYl6uZlfDw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-xml": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.6.0.tgz", + "integrity": "sha512-e7lX/dk//F6Qf7BB6PTY4+p2yuOQtyOeHGyapYHNwqSp2OnYpwQt49A/Nin2XmKBQ69pwagR4k/lQBq8lbHQkA==", + "license": "MIT", + "dependencies": { + "fast-xml-parser": "^5.5.9", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/logger": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.4.0.tgz", + "integrity": "sha512-rbAE25KUfjU/s3XHUdJgceoCP5dEOpMx85J04kF+QMdta73XkuG9JGHHinch+XIoKpBdqljin+KqURpJriSzLA==", + "license": "MIT", + "dependencies": { + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/storage-blob": { + "version": "12.33.0", + "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.33.0.tgz", + "integrity": "sha512-2SX8oP8PyblUcAFZSg39c8Ls+tFjavM6sBeV+qpw33mRzRhI/5hrFJmJ/x0H9xx5l6ECPvgSP8uPxqTeVbHNIA==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.3", + "@azure/core-http-compat": "^2.2.0", + "@azure/core-lro": "^2.2.0", + "@azure/core-paging": "^1.6.2", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/core-xml": "^1.4.5", + "@azure/logger": "^1.1.4", + "@azure/storage-common": "^12.4.1", + "events": "^3.0.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/storage-common": { + "version": "12.5.0", + "resolved": "https://registry.npmjs.org/@azure/storage-common/-/storage-common-12.5.0.tgz", + "integrity": "sha512-bttzuhQiCIwrkzjPDA+AtAR7dg19L/CC6ztcqJ5LfvWpXuys9mHp0UQ0udYnoUvv9SCT9KTR5kqFvFr0e6k0lQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-http-compat": "^2.4.0", + "@azure/core-rest-pipeline": "^1.24.0", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.1.4", + "events": "^3.3.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -5026,6 +5229,18 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@nodable/entities": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", + "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -7459,6 +7674,42 @@ "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", "license": "MIT" }, + "node_modules/@typespec/ts-http-runtime": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.9.tgz", + "integrity": "sha512-edSdeAqkdxBVzA1yL1LrLCml1YjyCVvPMtMqJpbF+6K609tHe8V6sQUzFQSGcYNhcuhOceZtjvN32+mpIth30A==", + "license": "MIT", + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@typespec/ts-http-runtime/node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/@typespec/ts-http-runtime/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/@ungap/structured-clone": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", @@ -7939,6 +8190,18 @@ "node": ">= 8" } }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/arch": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz", @@ -11840,6 +12103,45 @@ ], "license": "BSD-3-Clause" }, + "node_modules/fast-xml-builder": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.1.tgz", + "integrity": "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.11.1.tgz", + "integrity": "sha512-TBw6K/fxoQGGjCmZDw9w/ZwP3uDcnTM4YH/g+PFRWr8sbe5idXtxNN6vITh4+1ruCZaho6uBFurElsA7F0zzgw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.2", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastq": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", @@ -13024,6 +13326,28 @@ "node": ">=8.0.0" } }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/http-proxy-agent/node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/http-proxy-middleware": { "version": "2.0.10", "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.10.tgz", @@ -13662,6 +13986,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-unsafe": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.2.tgz", + "integrity": "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/is-wsl": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", @@ -17965,6 +18301,21 @@ "node": "^12.20.0 || ^14.13.1 || >=16.0.0" } }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", @@ -22109,6 +22460,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.2.tgz", + "integrity": "sha512-rDG3Ah4TV0k1hWvLSzkZtMmLN9+eS+h3knq4MP6A42Y3Yh5qGNnOUs1jJkoSr8FG5dsL28c7KgkIBzSEykqtuw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, "node_modules/style-to-js": { "version": "1.1.17", "resolved": "https://registry.npmjs.org/style-to-js/-/style-to-js-1.1.17.tgz", @@ -24038,6 +24404,21 @@ "xml-js": "bin/cli.js" } }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/xml-parser-xo": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/xml-parser-xo/-/xml-parser-xo-4.1.5.tgz", diff --git a/package.json b/package.json index 4b33b2382d..6c598530a7 100644 --- a/package.json +++ b/package.json @@ -23,10 +23,11 @@ "audit:generate": "node scripts/generate-audit-list.mjs", "kb:test:aa": "npm run kb:clean && npm run prestart && npm run start", "kb:prodtest:aa": "npm run kb:clean && npm run prebuild && npm run build && npm run serve", - "test:scripts": "bash scripts/test-slugify.sh && bash scripts/test-heading-skip.sh && bash scripts/test-md-extension-autofix.sh", + "test:scripts": "bash scripts/test-slugify.sh && bash scripts/test-heading-skip.sh && bash scripts/test-md-extension-autofix.sh && node --test scripts/test-set-blob-metadata.mjs", "prepare": "husky || true" }, "dependencies": { + "@azure/storage-blob": "^12.33.0", "@docusaurus/babel": "^3.10.2", "@docusaurus/core": "^3.10.2", "@docusaurus/faster": "^3.10.2", diff --git a/scripts/mint-container-sas.mjs b/scripts/mint-container-sas.mjs new file mode 100644 index 0000000000..3e9756bc49 --- /dev/null +++ b/scripts/mint-container-sas.mjs @@ -0,0 +1,51 @@ +#!/usr/bin/env node + +/** + * Print a short-lived container SAS token for azcopy, using the account key. + * + * Usage: + * node scripts/mint-container-sas.mjs [--container '$web'] [--hours 2] [--permissions dlrw] + * + * Reads STORAGE_ACCOUNT_NAME and STORAGE_ACCOUNT_KEY from the environment. + * Replaces `az storage container generate-sas`, which was the only remaining + * reason to install the Azure CLI on the deploy runner. + */ + +import { + ContainerSASPermissions, + StorageSharedKeyCredential, + generateBlobSASQueryParameters, +} from '@azure/storage-blob'; + +const args = { container: '$web', hours: '2', permissions: 'dlrw' }; +const argv = process.argv.slice(2); +for (let i = 0; i < argv.length; i += 2) { + const field = argv[i].replace(/^--/, ''); + if (!(field in args) || i + 1 >= argv.length) { + console.error(`usage: mint-container-sas.mjs [--container NAME] [--hours N] [--permissions dlrw]`); + process.exit(1); + } + args[field] = argv[i + 1]; +} + +const account = process.env.STORAGE_ACCOUNT_NAME; +const key = process.env.STORAGE_ACCOUNT_KEY; +if (!account || !key) { + console.error('error: STORAGE_ACCOUNT_NAME and STORAGE_ACCOUNT_KEY must be set'); + process.exit(1); +} + +const now = new Date(); +const sas = generateBlobSASQueryParameters( + { + containerName: args.container, + permissions: ContainerSASPermissions.parse(args.permissions), + // Five minutes of clock-skew allowance so a fast runner does not present a + // token whose start time the storage service still considers in the future. + startsOn: new Date(now.getTime() - 5 * 60 * 1000), + expiresOn: new Date(now.getTime() + Number(args.hours) * 60 * 60 * 1000), + }, + new StorageSharedKeyCredential(account, key), +); + +process.stdout.write(sas.toString()); diff --git a/scripts/set-blob-metadata.mjs b/scripts/set-blob-metadata.mjs new file mode 100644 index 0000000000..6a468cc8d7 --- /dev/null +++ b/scripts/set-blob-metadata.mjs @@ -0,0 +1,261 @@ +#!/usr/bin/env node + +/** + * Stamp `public_url` metadata and content types onto the static-site container. + * + * Usage: + * node scripts/set-blob-metadata.mjs \ + * --account NAME --key KEY --container '$web' --base-url https://docs.example.com + * + * Environment fallbacks: STORAGE_ACCOUNT_NAME, STORAGE_ACCOUNT_KEY, APP_EXTERNAL_URL + * Pass --force to rewrite every blob regardless of its current state. + * Pass --dry-run to list and plan without writing anything. + * + * Replaces two deploy steps that each walked the whole container: a per-blob + * `az storage blob metadata update` loop (~0.6s of CLI cold start each, 20 at a + * time, ~47 minutes for the site's ~86k blobs) and 19 `az storage blob + * update-batch` calls, one per file extension. Both are folded into a single + * listing pass here: `includeMetadata` returns each blob's metadata and + * properties inline, so deciding what needs writing costs no extra round trips, + * and only blobs that are actually missing or stale get a PUT. + */ + +import { BlobServiceClient, StorageSharedKeyCredential } from '@azure/storage-blob'; +import { pathToFileURL } from 'url'; + +const METADATA_KEY = 'public_url'; + +// Mirrors the extension/content-type pairs the previous `az storage blob +// update-batch` steps applied. Extensions absent from this map keep whatever +// content type azcopy inferred at upload, which is what update-batch did too. +const CONTENT_TYPE_BY_EXTENSION = new Map(Object.entries({ + css: 'text/css', + js: 'application/javascript', + mjs: 'application/javascript', + json: 'application/json', + html: 'text/html', + htm: 'text/html', + xml: 'application/xml', + txt: 'text/plain', + png: 'image/png', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + gif: 'image/gif', + webp: 'image/webp', + svg: 'image/svg+xml', + ico: 'image/x-icon', + woff: 'font/woff', + woff2: 'font/woff2', + ttf: 'font/ttf', + otf: 'font/otf', +})); + +// Azure sends metadata as `x-ms-meta-*` headers, which are latin-1 on the wire, +// so a raw blob name is not safe to interpolate: the site ships names carrying +// a Cyrillic homoglyph and a curly quote, and both throw at socket-write time. +// Percent-encode each path segment and leave the separators intact. +export function publicUrlFor(baseUrl, blobName) { + const encoded = blobName.split('/').map(encodeURIComponent).join('/'); + return `${baseUrl.replace(/\/+$/, '')}/${encoded}`; +} + +export function contentTypeFor(blobName) { + const base = blobName.slice(blobName.lastIndexOf('/') + 1); + const dot = base.lastIndexOf('.'); + if (dot <= 0) return undefined; + return CONTENT_TYPE_BY_EXTENSION.get(base.slice(dot + 1).toLowerCase()); +} + +/** + * Decide what a single blob needs. Returns null when it is already correct, + * so an unchanged deploy writes nothing. + */ +export function planFor(blob, baseUrl, force = false) { + const wantUrl = publicUrlFor(baseUrl, blob.name); + const wantType = contentTypeFor(blob.name); + const properties = blob.properties ?? {}; + + const needsMetadata = force || blob.metadata?.[METADATA_KEY] !== wantUrl; + const needsContentType = + wantType !== undefined && (force || properties.contentType !== wantType); + + if (!needsMetadata && !needsContentType) return null; + return { name: blob.name, wantUrl, wantType, needsMetadata, needsContentType, properties }; +} + +/** Run `tasks` with at most `limit` in flight, collecting rejections per task. */ +async function runPool(tasks, limit, onSettled) { + let next = 0; + const workers = Array.from({ length: Math.min(limit, tasks.length) }, async () => { + while (next < tasks.length) { + const index = next++; + try { + await tasks[index](); + onSettled(null); + } catch (error) { + onSettled(`${tasks[index].blobName}: ${error?.message ?? error}`); + } + } + }); + await Promise.all(workers); +} + +/** A bad invocation, reported as a one-line message rather than a stack trace. */ +class UsageError extends Error {} + +function parseArgs(argv) { + const args = { + account: process.env.STORAGE_ACCOUNT_NAME, + key: process.env.STORAGE_ACCOUNT_KEY, + container: '$web', + baseUrl: process.env.APP_EXTERNAL_URL ?? '', + force: false, + dryRun: false, + concurrency: 64, + }; + const flags = new Map([ + ['--account', 'account'], + ['--key', 'key'], + ['--container', 'container'], + ['--base-url', 'baseUrl'], + ['--concurrency', 'concurrency'], + ]); + + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--force') { + args.force = true; + continue; + } + if (argv[i] === '--dry-run') { + args.dryRun = true; + continue; + } + const field = flags.get(argv[i]); + if (!field) throw new UsageError(`unknown argument: ${argv[i]}`); + if (i + 1 >= argv.length) throw new UsageError(`${argv[i]} requires a value`); + args[field] = argv[++i]; + } + + args.concurrency = Number(args.concurrency); + const missing = ['account', 'key', 'baseUrl'].filter((field) => !args[field]); + if (missing.length) throw new UsageError(`missing required argument(s): ${missing.join(', ')}`); + if (!Number.isInteger(args.concurrency) || args.concurrency < 1) { + throw new UsageError('--concurrency must be a positive integer'); + } + return args; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + const client = new BlobServiceClient( + `https://${args.account}.blob.core.windows.net`, + new StorageSharedKeyCredential(args.account, args.key), + // The SDK's storage retry policy handles throttling (503) and transient + // resets with Azure-aware backoff, so the pool below does not retry itself. + { retryOptions: { maxTries: 5 } }, + ); + const container = client.getContainerClient(args.container); + + console.log(`Listing blobs in '${args.container}'...`); + let listed = 0; + const plans = []; + for await (const blob of container.listBlobsFlat({ includeMetadata: true })) { + listed++; + const plan = planFor(blob, args.baseUrl, args.force); + if (plan) plans.push(plan); + } + + const metadataWrites = plans.filter((plan) => plan.needsMetadata).length; + const contentTypeWrites = plans.filter((plan) => plan.needsContentType).length; + console.log( + ` ${listed} blobs listed, ${plans.length} need writes ` + + `(${metadataWrites} metadata, ${contentTypeWrites} content type)`, + ); + + if (!plans.length) { + console.log('All blobs already carry the correct metadata and content types.'); + return 0; + } + + if (args.dryRun) { + console.log('Dry run: sample of blobs that would be written:'); + for (const plan of plans.slice(0, 20)) { + const what = [plan.needsMetadata && 'metadata', plan.needsContentType && `type=${plan.wantType}`] + .filter(Boolean) + .join(', '); + console.log(` ${plan.name} (${what})`); + } + return 0; + } + + const tasks = plans.map((plan) => { + const task = async () => { + const blobClient = container.getBlobClient(plan.name); + if (plan.needsMetadata) { + // setMetadata replaces the whole metadata dict, matching the semantics + // of `az storage blob metadata update --metadata public_url=...`. + await blobClient.setMetadata({ [METADATA_KEY]: plan.wantUrl }); + } + if (plan.needsContentType) { + // setHTTPHeaders replaces every header it accepts, so carry the other + // content settings through rather than blanking them. + await blobClient.setHTTPHeaders({ + blobContentType: plan.wantType, + blobCacheControl: plan.properties.cacheControl, + blobContentEncoding: plan.properties.contentEncoding, + blobContentLanguage: plan.properties.contentLanguage, + blobContentDisposition: plan.properties.contentDisposition, + }); + } + }; + task.blobName = plan.name; + return task; + }); + + const started = Date.now(); + const errors = []; + let done = 0; + let failed = 0; + + await runPool(tasks, args.concurrency, (error) => { + if (error) { + failed++; + if (errors.length < 20) errors.push(error); + } else { + done++; + } + const processed = done + failed; + if (processed % 5000 === 0) { + const rate = processed / Math.max((Date.now() - started) / 1000, 1e-9); + console.log(` ${processed}/${tasks.length} (${rate.toFixed(0)}/s)`); + } + }); + + const elapsed = ((Date.now() - started) / 1000).toFixed(1); + console.log(`Updated ${done} blobs in ${elapsed}s (${failed} failed)`); + + if (errors.length) { + console.error('First failures:'); + for (const error of errors) console.error(` ${error}`); + } + return failed ? 1 : 0; +} + +// Only run when invoked directly, so the helpers above stay importable by tests. +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main() + .then((code) => process.exit(code)) + .catch((error) => { + if (error instanceof UsageError) { + console.error(`error: ${error.message}`); + console.error( + 'usage: node scripts/set-blob-metadata.mjs --account NAME --key KEY ' + + "[--container '$web'] --base-url URL [--force] [--dry-run] [--concurrency N]", + ); + } else { + console.error(error); + } + process.exit(1); + }); +} diff --git a/scripts/test-set-blob-metadata.mjs b/scripts/test-set-blob-metadata.mjs new file mode 100644 index 0000000000..dbb0bf882c --- /dev/null +++ b/scripts/test-set-blob-metadata.mjs @@ -0,0 +1,175 @@ +#!/usr/bin/env node + +/** + * Tests for scripts/set-blob-metadata.mjs + * + * Usage: + * node --test scripts/test-set-blob-metadata.mjs + * + * Covers the pure decision logic only: URL construction, content-type lookup, + * and the skip/write plan. The Azure calls themselves are exercised on deploy. + */ + +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { contentTypeFor, planFor, publicUrlFor } from './set-blob-metadata.mjs'; + +const BASE = 'https://docs.example.com'; + +test('publicUrlFor keeps path separators and joins to the base', () => { + assert.equal( + publicUrlFor(BASE, 'docs/pingcastle/index.html'), + 'https://docs.example.com/docs/pingcastle/index.html', + ); +}); + +test('publicUrlFor strips trailing slashes from the base', () => { + assert.equal(publicUrlFor(`${BASE}/`, 'a.html'), 'https://docs.example.com/a.html'); + assert.equal(publicUrlFor(`${BASE}///`, 'a.html'), 'https://docs.example.com/a.html'); +}); + +// These two names are real files in the built site and are exactly what broke +// the previous implementation: it interpolated them raw into an x-ms-meta-* +// header, which throws UnicodeEncodeError at socket-write time. +test('publicUrlFor encodes non-ASCII blob names to pure ASCII', () => { + const cyrillic = 'docs/kb/how_to_trigger_a_workflow_when_a_user_сreates_a_group/index.html'; + const curlyQuote = 'docs/kb/error_“collect_information/index.html'; + + for (const name of [cyrillic, curlyQuote]) { + const url = publicUrlFor(BASE, name); + assert.doesNotThrow(() => Buffer.from(url, 'latin1')); + // eslint-disable-next-line no-control-regex + assert.match(url, /^[\x00-\x7F]*$/, `expected ASCII-only output, got ${url}`); + } + + assert.equal( + publicUrlFor(BASE, cyrillic), + 'https://docs.example.com/docs/kb/how_to_trigger_a_workflow_when_a_user_%D1%81reates_a_group/index.html', + ); +}); + +test('publicUrlFor encodes spaces and URL-significant characters', () => { + assert.equal( + publicUrlFor(BASE, 'assets/images/ADMX Template-480a6d.png'), + 'https://docs.example.com/assets/images/ADMX%20Template-480a6d.png', + ); + // A literal '?' or '#' in a name would otherwise truncate the URL. + assert.equal(publicUrlFor(BASE, 'a/b?c#d.html'), 'https://docs.example.com/a/b%3Fc%23d.html'); +}); + +test('contentTypeFor maps the extensions the az update-batch steps covered', () => { + const expected = { + 'a.css': 'text/css', + 'a.js': 'application/javascript', + 'a.mjs': 'application/javascript', + 'a.json': 'application/json', + 'a.html': 'text/html', + 'a.htm': 'text/html', + 'a.xml': 'application/xml', + 'a.txt': 'text/plain', + 'a.png': 'image/png', + 'a.jpg': 'image/jpeg', + 'a.jpeg': 'image/jpeg', + 'a.gif': 'image/gif', + 'a.webp': 'image/webp', + 'a.svg': 'image/svg+xml', + 'a.ico': 'image/x-icon', + 'a.woff': 'font/woff', + 'a.woff2': 'font/woff2', + 'a.ttf': 'font/ttf', + 'a.otf': 'font/otf', + }; + for (const [name, type] of Object.entries(expected)) { + assert.equal(contentTypeFor(name), type, `${name} should map to ${type}`); + } + assert.equal(Object.keys(expected).length, 19, 'should cover all 19 original patterns'); +}); + +test('contentTypeFor is case-insensitive and ignores unknown or absent extensions', () => { + assert.equal(contentTypeFor('a/B.WEBP'), 'image/webp'); + assert.equal(contentTypeFor('a/README'), undefined); + assert.equal(contentTypeFor('a/archive.tar.gz'), undefined); + // A dotfile is a name, not an extension. + assert.equal(contentTypeFor('a/.nojekyll'), undefined); + // A dot in a directory must not be read as the file's extension. + assert.equal(contentTypeFor('docs/accessanalyzer/12.0/index'), undefined); +}); + +test('planFor returns null when metadata and content type are both current', () => { + const blob = { + name: 'a.html', + metadata: { public_url: `${BASE}/a.html` }, + properties: { contentType: 'text/html' }, + }; + assert.equal(planFor(blob, BASE), null); +}); + +test('planFor flags a missing or stale public_url', () => { + const properties = { contentType: 'text/html' }; + + const missing = planFor({ name: 'a.html', metadata: {}, properties }, BASE); + assert.equal(missing.needsMetadata, true); + assert.equal(missing.needsContentType, false); + + const stale = planFor( + { name: 'a.html', metadata: { public_url: 'https://old.example.com/a.html' }, properties }, + BASE, + ); + assert.equal(stale.needsMetadata, true); + assert.equal(stale.wantUrl, `${BASE}/a.html`); +}); + +test('planFor flags a wrong content type independently of metadata', () => { + const plan = planFor( + { + name: 'a.webp', + metadata: { public_url: `${BASE}/a.webp` }, + properties: { contentType: 'application/octet-stream' }, + }, + BASE, + ); + assert.equal(plan.needsMetadata, false); + assert.equal(plan.needsContentType, true); + assert.equal(plan.wantType, 'image/webp'); +}); + +test('planFor leaves unmapped extensions alone rather than blanking them', () => { + const plan = planFor( + { + name: 'a.pdf', + metadata: { public_url: `${BASE}/a.pdf` }, + properties: { contentType: 'application/pdf' }, + }, + BASE, + ); + assert.equal(plan, null); +}); + +test('planFor with --force rewrites even a fully current blob', () => { + const blob = { + name: 'a.html', + metadata: { public_url: `${BASE}/a.html` }, + properties: { contentType: 'text/html' }, + }; + const plan = planFor(blob, BASE, true); + assert.equal(plan.needsMetadata, true); + assert.equal(plan.needsContentType, true); +}); + +test('planFor carries other content settings through for setHTTPHeaders', () => { + const plan = planFor( + { + name: 'a.html', + metadata: {}, + properties: { contentType: 'text/plain', cacheControl: 'max-age=600' }, + }, + BASE, + ); + assert.equal(plan.properties.cacheControl, 'max-age=600'); +}); + +test('planFor tolerates a blob listed without metadata', () => { + const plan = planFor({ name: 'a.html', properties: { contentType: 'text/html' } }, BASE); + assert.equal(plan.needsMetadata, true); +}); From e889eb482fde8640dbc3ab0b9d6d68270aabeb4d Mon Sep 17 00:00:00 2001 From: thobed <10742470+thobed@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:31:11 -0400 Subject: [PATCH 2/3] test(ci): add Azurite integration test for blob metadata and SAS scripts set-blob-metadata.mjs gains --endpoint (or AZURE_BLOB_ENDPOINT) so it can be pointed at the Azurite emulator. The integration test seeds a container with the blob names that broke the old az loop (Cyrillic homoglyph, space), wrong and unmapped content types, and stale/current metadata, then checks dry-run writes nothing, a real run fixes exactly the stale blobs while preserving cache-control, a second run is a no-op, and a SAS minted by mint-container-sas.mjs can list/write/delete while a list-only SAS is refused. Skips cleanly when no emulator is listening. --- package.json | 2 +- scripts/set-blob-metadata.mjs | 7 +- .../test-set-blob-metadata-integration.mjs | 103 ++++++++++++++++++ 3 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 scripts/test-set-blob-metadata-integration.mjs diff --git a/package.json b/package.json index 6c598530a7..9210068c44 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "audit:generate": "node scripts/generate-audit-list.mjs", "kb:test:aa": "npm run kb:clean && npm run prestart && npm run start", "kb:prodtest:aa": "npm run kb:clean && npm run prebuild && npm run build && npm run serve", - "test:scripts": "bash scripts/test-slugify.sh && bash scripts/test-heading-skip.sh && bash scripts/test-md-extension-autofix.sh && node --test scripts/test-set-blob-metadata.mjs", + "test:scripts": "bash scripts/test-slugify.sh && bash scripts/test-heading-skip.sh && bash scripts/test-md-extension-autofix.sh && node --test scripts/test-set-blob-metadata.mjs && node scripts/test-set-blob-metadata-integration.mjs", "prepare": "husky || true" }, "dependencies": { diff --git a/scripts/set-blob-metadata.mjs b/scripts/set-blob-metadata.mjs index 6a468cc8d7..b142f06971 100644 --- a/scripts/set-blob-metadata.mjs +++ b/scripts/set-blob-metadata.mjs @@ -112,6 +112,7 @@ function parseArgs(argv) { force: false, dryRun: false, concurrency: 64, + endpoint: process.env.AZURE_BLOB_ENDPOINT, }; const flags = new Map([ ['--account', 'account'], @@ -119,6 +120,7 @@ function parseArgs(argv) { ['--container', 'container'], ['--base-url', 'baseUrl'], ['--concurrency', 'concurrency'], + ['--endpoint', 'endpoint'], ]); for (let i = 0; i < argv.length; i++) { @@ -148,8 +150,9 @@ function parseArgs(argv) { async function main() { const args = parseArgs(process.argv.slice(2)); + // --endpoint exists so the script can be pointed at the Azurite emulator. const client = new BlobServiceClient( - `https://${args.account}.blob.core.windows.net`, + args.endpoint ?? `https://${args.account}.blob.core.windows.net`, new StorageSharedKeyCredential(args.account, args.key), // The SDK's storage retry policy handles throttling (503) and transient // resets with Azure-aware backoff, so the pool below does not retry itself. @@ -251,7 +254,7 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) console.error(`error: ${error.message}`); console.error( 'usage: node scripts/set-blob-metadata.mjs --account NAME --key KEY ' + - "[--container '$web'] --base-url URL [--force] [--dry-run] [--concurrency N]", + "[--container '$web'] --base-url URL [--force] [--dry-run] [--concurrency N] [--endpoint URL]", ); } else { console.error(error); diff --git a/scripts/test-set-blob-metadata-integration.mjs b/scripts/test-set-blob-metadata-integration.mjs new file mode 100644 index 0000000000..800ffd74c8 --- /dev/null +++ b/scripts/test-set-blob-metadata-integration.mjs @@ -0,0 +1,103 @@ +#!/usr/bin/env node + +/** + * Integration test for scripts/set-blob-metadata.mjs and + * scripts/mint-container-sas.mjs against the Azurite blob emulator. + * + * Usage: + * npx --package azurite azurite-blob --silent --location /tmp/azurite & + * node scripts/test-set-blob-metadata-integration.mjs + * + * Skips (exit 0) when nothing is listening on the emulator port, so it is + * safe to include in a suite that runs without Azurite. + */ + +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { BlobServiceClient, ContainerClient, StorageSharedKeyCredential } from '@azure/storage-blob'; + +const ACCOUNT = 'devstoreaccount1'; +const KEY = 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=='; +const ENDPOINT = `http://127.0.0.1:10000/${ACCOUNT}`; +const BASE = 'https://docs.example.com'; +const CONTAINER = '$web'; +const probe = await fetch(`${ENDPOINT}/?comp=list`).catch(() => null); +if (!probe) { + console.log(`Azurite not reachable at ${ENDPOINT}, skipping integration test`); + process.exit(0); +} + +const env = { ...process.env, STORAGE_ACCOUNT_NAME: ACCOUNT, STORAGE_ACCOUNT_KEY: KEY }; + +const cred = new StorageSharedKeyCredential(ACCOUNT, KEY); +const service = new BlobServiceClient(ENDPOINT, cred); +const container = service.getContainerClient(CONTAINER); +await container.deleteIfExists(); +await container.create(); + +const cyrillic = 'docs/kb/how_to_trigger_a_workflow_when_a_user_сreates_a_group/index.html'; +const seed = [ + ['index.html', 'text/plain', undefined], + [cyrillic, 'text/html', undefined], + ['assets/images/ADMX Template-480a6d.png', 'application/octet-stream', undefined], + ['a.pdf', 'application/pdf', undefined], + ['x.webp', 'image/webp', undefined], + ['ok.css', 'text/css', `${BASE}/ok.css`], + ['stale.js', 'application/javascript', 'https://old.example.com/stale.js'], +]; +for (const [name, type, url] of seed) { + await container.getBlockBlobClient(name).upload('x', 1, { + blobHTTPHeaders: { blobContentType: type, blobCacheControl: 'max-age=600' }, + metadata: url ? { public_url: url } : undefined, + }); +} + +const run = (...extra) => + execFileSync('node', ['scripts/set-blob-metadata.mjs', '--container', CONTAINER, '--base-url', BASE, '--endpoint', ENDPOINT, ...extra], { env, encoding: 'utf8' }); + +let out = run('--dry-run'); +console.log(out); +assert.match(out, /7 blobs listed, 6 need writes \(6 metadata, 2 content type\)/); +assert.equal((await container.getBlobClient('index.html').getProperties()).contentType, 'text/plain', 'dry run must not write'); + +out = run(); +console.log(out); +assert.match(out, /Updated 6 blobs in [\d.]+s \(0 failed\)/); + +const expect = { + 'index.html': 'text/html', + [cyrillic]: 'text/html', + 'assets/images/ADMX Template-480a6d.png': 'image/png', + 'a.pdf': 'application/pdf', + 'x.webp': 'image/webp', + 'ok.css': 'text/css', + 'stale.js': 'application/javascript', +}; +for (const [name, type] of Object.entries(expect)) { + const p = await container.getBlobClient(name).getProperties(); + assert.equal(p.contentType, type, `${name} content type`); + assert.equal(p.cacheControl, 'max-age=600', `${name} cache-control preserved`); + assert.equal(p.metadata.public_url, name.split('/').map(encodeURIComponent).join('/').replace(/^/, `${BASE}/`), `${name} public_url`); +} + +out = run(); +console.log(out); +assert.match(out, /7 blobs listed, 0 need writes/); +assert.match(out, /All blobs already carry/); + +out = run('--force', '--dry-run'); +assert.match(out, /7 need writes \(7 metadata, 6 content type\)/); + +// SAS: mint with the script, then use it (no account key) to list and write. +const sas = execFileSync('node', ['scripts/mint-container-sas.mjs', '--container', CONTAINER, '--hours', '1', '--permissions', 'dlrw'], { env, encoding: 'utf8' }); +assert.match(sas, /sp=rwdl/); +const viaSas = new ContainerClient(`${ENDPOINT}/${CONTAINER}?${sas}`); +let n = 0; +for await (const _ of viaSas.listBlobsFlat()) n++; +assert.equal(n, 7, 'SAS can list'); +await viaSas.getBlockBlobClient('sas-written.txt').upload('y', 1); +await viaSas.deleteBlob('sas-written.txt'); +const readOnly = execFileSync('node', ['scripts/mint-container-sas.mjs', '--container', CONTAINER, '--permissions', 'l'], { env, encoding: 'utf8' }); +await assert.rejects(new ContainerClient(`${ENDPOINT}/${CONTAINER}?${readOnly}`).getBlockBlobClient('nope.txt').upload('y', 1), (e) => e.statusCode === 403); + +console.log('INTEGRATION: all assertions passed'); From 63bef150766f0bc850a9509629d51060e47c44b4 Mon Sep 17 00:00:00 2001 From: thobed <10742470+thobed@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:52:05 -0400 Subject: [PATCH 3/3] ci: install dependencies before running script tests The test-scripts job ran npm run test:scripts on a bare checkout. That worked while every test was a shell script, but the blob metadata tests import @azure/storage-blob. --- .github/workflows/vale-autofix.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/vale-autofix.yml b/.github/workflows/vale-autofix.yml index a51ef27809..6838838302 100644 --- a/.github/workflows/vale-autofix.yml +++ b/.github/workflows/vale-autofix.yml @@ -38,6 +38,17 @@ jobs: echo "changed=true" >> "$GITHUB_OUTPUT" fi + - name: Set up Node.js + if: steps.scripts-changed.outputs.changed == 'true' + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22.x' + cache: npm + + - name: Install dependencies + if: steps.scripts-changed.outputs.changed == 'true' + run: npm ci --ignore-scripts + - name: Test anchor/slugify scripts if: steps.scripts-changed.outputs.changed == 'true' run: npm run test:scripts