diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index 4fc5345c8b..02086073ad 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -40,7 +40,10 @@ jobs: echo "environment=development" >> $GITHUB_OUTPUT fi - build: + # Build and deploy share one job so the 1.5 GB build/ directory never makes a + # round trip through the artifact store. Handing it between two jobs cost about + # 4.5 minutes (1.5 min upload + 3 min download) purely in transfer. + build-and-deploy: runs-on: gh-runner-large needs: determine-environment environment: ${{ needs.determine-environment.outputs.environment }} @@ -69,185 +72,114 @@ jobs: restore-keys: | v2-${{ runner.os }}-node- - - name: Cache Docusaurus build - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - id: cache-build - with: - path: | - .docusaurus - key: v2-${{ runner.os }}-docusaurus-${{ hashFiles('src/**', 'docs/**', 'blog/**', 'docusaurus.config.js', 'sidebars.js') }} - restore-keys: | - v2-${{ runner.os }}-docusaurus- - - - name: Cache webpack + # Rspack's persistent cache (future.faster.rspackPersistentCache) and the + # MDX cross-compiler cache both live here, and this is the cache that + # actually shortens a rebuild. The key carries github.run_id so every run + # misses on the exact key and therefore *saves* a refreshed cache, while + # restore-keys pull the newest previous one. Keying on package-lock.json + # alone (the previous behaviour) meant the first run saved a cache that + # then hit forever and never picked up newly compiled modules. + # + # The former `.docusaurus` cache was dropped: its key hashed docs/**, so + # any content change missed it, and the restore-keys fallback just + # restored a stale directory that Docusaurus regenerates anyway. + - name: Cache Rspack/MDX compiler output uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: node_modules/.cache - key: v2-${{ runner.os }}-webpack-${{ hashFiles('**/package-lock.json') }} + key: v3-${{ runner.os }}-rspack-${{ hashFiles('**/package-lock.json') }}-${{ github.run_id }} restore-keys: | - v2-${{ runner.os }}-webpack- + v3-${{ runner.os }}-rspack-${{ hashFiles('**/package-lock.json') }}- + v3-${{ runner.os }}-rspack- - name: Install dependencies and build site run: | npm ci - - if [[ "${{ steps.cache-build.outputs.cache-hit }}" == "true" ]]; then - echo "Build cache found, checking if rebuild needed..." - else - echo "No build cache found, performing full build..." - fi - npm run ci env: NODE_OPTIONS: "--max-old-space-size=16384" NODE_ENV: ${{ needs.determine-environment.outputs.environment }} + # PR runs stop before the deploy steps below, so the built site is only + # reachable as an artifact. Kept PR-only: on push the build is deployed, + # and uploading 1.5GB there cost ~4.5min for nothing. - name: Upload artifact for deployment + if: github.event_name == 'pull_request' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: build-output path: build/ - deploy: - runs-on: ubuntu-latest - if: github.event_name != 'pull_request' - needs: [build, determine-environment] - environment: ${{ needs.determine-environment.outputs.environment }} - steps: - - name: Download build artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: build-output - path: build/ - - - name: Install azcopy - run: | - wget -O azcopy.tar.gz https://aka.ms/downloadazcopy-v10-linux - tar -xf azcopy.tar.gz --strip-components=1 - sudo mv azcopy /usr/local/bin/ - azcopy --version - - - name: Install Azure CLI - run: | - if ! command -v az &> /dev/null; then - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - fi - az version - - - name: Upload to Azure Blob Storage with AzCopy - run: | - echo "Deploying to ${{ needs.determine-environment.outputs.environment }} environment" - echo "Starting sync of changed files..." - - # Create SAS token for azcopy (using account key) - end_date=$(date -u -d "2 hours" '+%Y-%m-%dT%H:%MZ') - sas_token=$(az storage container generate-sas \ - --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} \ - --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} \ - --name '$web' \ - --permissions dlrw \ - --expiry $end_date \ - --output tsv) - - azcopy sync "./build/" \ - "https://${{ secrets.STORAGE_ACCOUNT_NAME }}.blob.core.windows.net/\$web?$sas_token" \ - --delete-destination=true \ - --log-level=INFO \ - --cap-mbps=0 \ - --block-size-mb=4 - - echo "Sync completed!" - - - name: Set MIME types for all file types - run: | - echo "Setting MIME types for all file types..." - - # Web files - echo "Setting MIME types for web files..." - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.css" --content-type "text/css" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.js" --content-type "application/javascript" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.mjs" --content-type "application/javascript" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.json" --content-type "application/json" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.html" --content-type "text/html" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.htm" --content-type "text/html" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.xml" --content-type "application/xml" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.txt" --content-type "text/plain" --no-progress || true - - # Images - echo "Setting MIME types for images..." - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.png" --content-type "image/png" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.jpg" --content-type "image/jpeg" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.jpeg" --content-type "image/jpeg" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.gif" --content-type "image/gif" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.webp" --content-type "image/webp" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.svg" --content-type "image/svg+xml" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.ico" --content-type "image/x-icon" --no-progress || true - - # Fonts - echo "Setting MIME types for fonts..." - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.woff" --content-type "font/woff" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.woff2" --content-type "font/woff2" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.ttf" --content-type "font/ttf" --no-progress || true - az storage blob update-batch --account-name ${{ secrets.STORAGE_ACCOUNT_NAME }} --account-key ${{ secrets.STORAGE_ACCOUNT_KEY }} --source '$web' --pattern "*.otf" --content-type "font/otf" --no-progress || true - - echo "All MIME types set successfully!" - - - name: Set public URL metadata on blobs - env: - APP_EXTERNAL_URL: ${{ secrets.APP_EXTERNAL_URL || vars.APP_EXTERNAL_URL }} - run: | - if [[ -z "$APP_EXTERNAL_URL" ]]; then - echo "APP_EXTERNAL_URL not configured, skipping public URL metadata" - exit 0 - fi - PUBLIC_URL="${APP_EXTERNAL_URL%/}" - - echo "Setting public_url metadata on blobs (base: $PUBLIC_URL)..." - - ACCOUNT="${{ secrets.STORAGE_ACCOUNT_NAME }}" - KEY="${{ secrets.STORAGE_ACCOUNT_KEY }}" - - blob_count=0 - az storage blob list \ - --account-name "$ACCOUNT" \ - --account-key "$KEY" \ - --container-name '$web' \ - --query "[].name" -o tsv | \ - while IFS= read -r blob; do - az storage blob metadata update \ - --account-name "$ACCOUNT" \ - --account-key "$KEY" \ - --container-name '$web' \ - --name "$blob" \ - --metadata "public_url=${PUBLIC_URL}/${blob}" \ - --output none & - - blob_count=$((blob_count + 1)) - # Run up to 20 concurrent updates, then wait for the batch - if (( blob_count % 20 == 0 )); then - wait - echo " Processed $blob_count blobs..." + # Installed into the job's temp dir and put on PATH, so this needs no + # sudo. The self-hosted runner's sudo availability is not something the + # workflow should depend on. + - name: Install azcopy + if: github.event_name != 'pull_request' + run: | + mkdir -p "$RUNNER_TEMP/azcopy" + curl -fsSL -o "$RUNNER_TEMP/azcopy/azcopy.tar.gz" https://aka.ms/downloadazcopy-v10-linux + tar -xf "$RUNNER_TEMP/azcopy/azcopy.tar.gz" -C "$RUNNER_TEMP/azcopy" --strip-components=1 + echo "$RUNNER_TEMP/azcopy" >> "$GITHUB_PATH" + "$RUNNER_TEMP/azcopy/azcopy" --version + + # The SAS token is minted with the storage SDK already in node_modules, + # so the Azure CLI is no longer installed on the runner at all. + - name: Upload to Azure Blob Storage with AzCopy + if: github.event_name != 'pull_request' + env: + STORAGE_ACCOUNT_NAME: ${{ secrets.STORAGE_ACCOUNT_NAME }} + STORAGE_ACCOUNT_KEY: ${{ secrets.STORAGE_ACCOUNT_KEY }} + run: | + echo "Deploying to ${{ needs.determine-environment.outputs.environment }} environment" + echo "Starting sync of changed files..." + + sas_token=$(node scripts/mint-container-sas.mjs --container '$web' --hours 2 --permissions dlrw) + + azcopy sync "./build/" \ + "https://${STORAGE_ACCOUNT_NAME}.blob.core.windows.net/\$web?${sas_token}" \ + --delete-destination=true \ + --log-level=INFO \ + --cap-mbps=0 \ + --block-size-mb=4 + + echo "Sync completed!" + + # Folds together two steps that each walked the whole container: a per-blob + # `az storage blob metadata update` loop (~0.6s of CLI cold start each, 20 at + # a time, ~47 minutes for the site's ~86k blobs) and 19 `az storage blob + # update-batch` calls, one per extension. The script lists metadata and + # properties inline with the blob listing and writes only what is absent or + # stale. Pass --force to rewrite every blob unconditionally. + - name: Set public URL metadata and content types on blobs + if: github.event_name != 'pull_request' + env: + STORAGE_ACCOUNT_NAME: ${{ secrets.STORAGE_ACCOUNT_NAME }} + STORAGE_ACCOUNT_KEY: ${{ secrets.STORAGE_ACCOUNT_KEY }} + run: | + if [[ -z "$APP_EXTERNAL_URL" ]]; then + echo "APP_EXTERNAL_URL not configured, skipping blob metadata" + exit 0 fi - done - wait - echo "Public URL metadata set on all blobs" + node scripts/set-blob-metadata.mjs --container '$web' - - name: Purge CDN endpoint (if configured) - run: | - if [[ -n "${{ secrets.CDN_ENDPOINT_NAME }}" ]] && [[ -n "${{ secrets.CDN_PROFILE_NAME }}" ]] && [[ -n "${{ secrets.CDN_RESOURCE_GROUP }}" ]]; then - echo "Note: CDN purge requires Azure login. Skipping CDN purge when using storage key authentication." - echo "To use CDN purge, you'll need to use Azure AD authentication or purge CDN manually." - else - echo "CDN configuration not found, skipping CDN purge." - fi + - name: Purge CDN endpoint (if configured) + if: github.event_name != 'pull_request' + run: | + if [[ -n "${{ secrets.CDN_ENDPOINT_NAME }}" ]] && [[ -n "${{ secrets.CDN_PROFILE_NAME }}" ]] && [[ -n "${{ secrets.CDN_RESOURCE_GROUP }}" ]]; then + echo "Note: CDN purge requires Azure login. Skipping CDN purge when using storage key authentication." + echo "To use CDN purge, you'll need to use Azure AD authentication or purge CDN manually." + else + echo "CDN configuration not found, skipping CDN purge." + fi - - name: Display deployment URL - run: | - echo "Deployment complete!" - echo "Environment: ${{ needs.determine-environment.outputs.environment }}" - echo "URL: https://${{ secrets.STORAGE_ACCOUNT_NAME }}.z13.web.core.windows.net" - if [[ -n "${{ secrets.CUSTOM_DOMAIN }}" ]]; then - echo "Custom Domain: ${{ secrets.CUSTOM_DOMAIN }}" - fi - echo "All files deployed with proper MIME types for optimal browser compatibility!" + - name: Display deployment URL + if: github.event_name != 'pull_request' + run: | + echo "Deployment complete!" + echo "Environment: ${{ needs.determine-environment.outputs.environment }}" + echo "URL: https://${{ secrets.STORAGE_ACCOUNT_NAME }}.z13.web.core.windows.net" + if [[ -n "${{ secrets.CUSTOM_DOMAIN }}" ]]; then + echo "Custom Domain: ${{ secrets.CUSTOM_DOMAIN }}" + fi + echo "All files deployed with proper MIME types for optimal browser compatibility!" diff --git a/.github/workflows/vale-autofix.yml b/.github/workflows/vale-autofix.yml index a51ef27809..6838838302 100644 --- a/.github/workflows/vale-autofix.yml +++ b/.github/workflows/vale-autofix.yml @@ -38,6 +38,17 @@ jobs: echo "changed=true" >> "$GITHUB_OUTPUT" fi + - name: Set up Node.js + if: steps.scripts-changed.outputs.changed == 'true' + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22.x' + cache: npm + + - name: Install dependencies + if: steps.scripts-changed.outputs.changed == 'true' + run: npm ci --ignore-scripts + - name: Test anchor/slugify scripts if: steps.scripts-changed.outputs.changed == 'true' run: npm run test:scripts diff --git a/docusaurus.config.js b/docusaurus.config.js index 5f6cf45ce7..9833ce2c5e 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -127,7 +127,9 @@ const config = { rspackBundler: true, rspackPersistentCache: true, mdxCrossCompilerCache: true, - ssgWorkerThreads: false, + // Renders the static pages across a worker pool instead of one thread. + // Requires future.v4.removeLegacyPostBuildHeadAttribute, set below. + ssgWorkerThreads: true, }, v4: { removeLegacyPostBuildHeadAttribute: true, diff --git a/package-lock.json b/package-lock.json index 26cc40d6f8..e536975db5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "netwrix-docs", "version": "0.1", "dependencies": { + "@azure/storage-blob": "^12.33.0", "@docusaurus/babel": "^3.10.2", "@docusaurus/core": "^3.10.2", "@docusaurus/faster": "^3.10.2", @@ -318,6 +319,208 @@ "@types/json-schema": "^7.0.15" } }, + "node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-auth": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.11.0.tgz", + "integrity": "sha512-IUZydyTUkDnYdstOW9pFOOUQlBjAepK5teihDE3x6yxsPJs/hsAaaYpeGxdxrgtOiJbBKSjKW7MDk7AEhb4LRg==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-client": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.11.1.tgz", + "integrity": "sha512-2QygG2F76ZpMP2eMztiJvAiFMu71M9rDeU7vO/QKg5Css7MgM4frUOslFjhVjRhbGaCNPtz/S8M6y46/fFKVuQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-http-compat": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.5.0.tgz", + "integrity": "sha512-BoSmXPx2er1Ai+wKlDvj29jIQespCNBwEmKyZVHO2kEFsWbGjAjwMCGzug3DJM5/QYIV3vej0S1zcU5bq9fa8w==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "@azure/core-client": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0" + } + }, + "node_modules/@azure/core-lro": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.7.2.tgz", + "integrity": "sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-util": "^1.2.0", + "@azure/logger": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-paging": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.7.0.tgz", + "integrity": "sha512-7GEAoIsaoBr6KELNRb8nypowCqvk8dnCHFCYg4XD4lOQGY2GqjQg5IhkRjyBFRO18CGSMq05PaNqSOE9GQro3g==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.25.0.tgz", + "integrity": "sha512-bMs8ekJLjX8wPV+9IPBges1SLPyuDtE9g5gLDWOpxzKcoOFQnpLGkbcT1tdw3FaAmDS1gnPmMmJ6y/T5B96kIA==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-tracing": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.4.0.tgz", + "integrity": "sha512-eGwxD0AtncrxeBM4tG8R55Pc3rdX1hNW2WibJAgYpCVA6E93mvvVH+LcssoVjOBrSKWS55yEIHsk0X8ctHmfOQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-util": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.14.0.tgz", + "integrity": "sha512-9n2pWK61veAuN0V20t9lOuoV4CFMdyAZ1ygZzvBGk/pBBJRib/PjL9PLXa/aI2CcPpyHfqVsxxqLCYl6uZlfDw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-xml": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.6.0.tgz", + "integrity": "sha512-e7lX/dk//F6Qf7BB6PTY4+p2yuOQtyOeHGyapYHNwqSp2OnYpwQt49A/Nin2XmKBQ69pwagR4k/lQBq8lbHQkA==", + "license": "MIT", + "dependencies": { + "fast-xml-parser": "^5.5.9", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/logger": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.4.0.tgz", + "integrity": "sha512-rbAE25KUfjU/s3XHUdJgceoCP5dEOpMx85J04kF+QMdta73XkuG9JGHHinch+XIoKpBdqljin+KqURpJriSzLA==", + "license": "MIT", + "dependencies": { + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/storage-blob": { + "version": "12.33.0", + "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.33.0.tgz", + "integrity": "sha512-2SX8oP8PyblUcAFZSg39c8Ls+tFjavM6sBeV+qpw33mRzRhI/5hrFJmJ/x0H9xx5l6ECPvgSP8uPxqTeVbHNIA==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.3", + "@azure/core-http-compat": "^2.2.0", + "@azure/core-lro": "^2.2.0", + "@azure/core-paging": "^1.6.2", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/core-xml": "^1.4.5", + "@azure/logger": "^1.1.4", + "@azure/storage-common": "^12.4.1", + "events": "^3.0.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/storage-common": { + "version": "12.5.0", + "resolved": "https://registry.npmjs.org/@azure/storage-common/-/storage-common-12.5.0.tgz", + "integrity": "sha512-bttzuhQiCIwrkzjPDA+AtAR7dg19L/CC6ztcqJ5LfvWpXuys9mHp0UQ0udYnoUvv9SCT9KTR5kqFvFr0e6k0lQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-http-compat": "^2.4.0", + "@azure/core-rest-pipeline": "^1.24.0", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.1.4", + "events": "^3.3.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -5026,6 +5229,18 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@nodable/entities": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", + "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -7459,6 +7674,42 @@ "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", "license": "MIT" }, + "node_modules/@typespec/ts-http-runtime": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.9.tgz", + "integrity": "sha512-edSdeAqkdxBVzA1yL1LrLCml1YjyCVvPMtMqJpbF+6K609tHe8V6sQUzFQSGcYNhcuhOceZtjvN32+mpIth30A==", + "license": "MIT", + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@typespec/ts-http-runtime/node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/@typespec/ts-http-runtime/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/@ungap/structured-clone": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", @@ -7939,6 +8190,18 @@ "node": ">= 8" } }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/arch": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz", @@ -11840,6 +12103,45 @@ ], "license": "BSD-3-Clause" }, + "node_modules/fast-xml-builder": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.1.tgz", + "integrity": "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.11.1.tgz", + "integrity": "sha512-TBw6K/fxoQGGjCmZDw9w/ZwP3uDcnTM4YH/g+PFRWr8sbe5idXtxNN6vITh4+1ruCZaho6uBFurElsA7F0zzgw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.2", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastq": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", @@ -13024,6 +13326,28 @@ "node": ">=8.0.0" } }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/http-proxy-agent/node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/http-proxy-middleware": { "version": "2.0.10", "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.10.tgz", @@ -13662,6 +13986,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-unsafe": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.2.tgz", + "integrity": "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/is-wsl": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", @@ -17965,6 +18301,21 @@ "node": "^12.20.0 || ^14.13.1 || >=16.0.0" } }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", @@ -22109,6 +22460,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.2.tgz", + "integrity": "sha512-rDG3Ah4TV0k1hWvLSzkZtMmLN9+eS+h3knq4MP6A42Y3Yh5qGNnOUs1jJkoSr8FG5dsL28c7KgkIBzSEykqtuw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, "node_modules/style-to-js": { "version": "1.1.17", "resolved": "https://registry.npmjs.org/style-to-js/-/style-to-js-1.1.17.tgz", @@ -24038,6 +24404,21 @@ "xml-js": "bin/cli.js" } }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/xml-parser-xo": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/xml-parser-xo/-/xml-parser-xo-4.1.5.tgz", diff --git a/package.json b/package.json index 4b33b2382d..9210068c44 100644 --- a/package.json +++ b/package.json @@ -23,10 +23,11 @@ "audit:generate": "node scripts/generate-audit-list.mjs", "kb:test:aa": "npm run kb:clean && npm run prestart && npm run start", "kb:prodtest:aa": "npm run kb:clean && npm run prebuild && npm run build && npm run serve", - "test:scripts": "bash scripts/test-slugify.sh && bash scripts/test-heading-skip.sh && bash scripts/test-md-extension-autofix.sh", + "test:scripts": "bash scripts/test-slugify.sh && bash scripts/test-heading-skip.sh && bash scripts/test-md-extension-autofix.sh && node --test scripts/test-set-blob-metadata.mjs && node scripts/test-set-blob-metadata-integration.mjs", "prepare": "husky || true" }, "dependencies": { + "@azure/storage-blob": "^12.33.0", "@docusaurus/babel": "^3.10.2", "@docusaurus/core": "^3.10.2", "@docusaurus/faster": "^3.10.2", diff --git a/scripts/mint-container-sas.mjs b/scripts/mint-container-sas.mjs new file mode 100644 index 0000000000..3e9756bc49 --- /dev/null +++ b/scripts/mint-container-sas.mjs @@ -0,0 +1,51 @@ +#!/usr/bin/env node + +/** + * Print a short-lived container SAS token for azcopy, using the account key. + * + * Usage: + * node scripts/mint-container-sas.mjs [--container '$web'] [--hours 2] [--permissions dlrw] + * + * Reads STORAGE_ACCOUNT_NAME and STORAGE_ACCOUNT_KEY from the environment. + * Replaces `az storage container generate-sas`, which was the only remaining + * reason to install the Azure CLI on the deploy runner. + */ + +import { + ContainerSASPermissions, + StorageSharedKeyCredential, + generateBlobSASQueryParameters, +} from '@azure/storage-blob'; + +const args = { container: '$web', hours: '2', permissions: 'dlrw' }; +const argv = process.argv.slice(2); +for (let i = 0; i < argv.length; i += 2) { + const field = argv[i].replace(/^--/, ''); + if (!(field in args) || i + 1 >= argv.length) { + console.error(`usage: mint-container-sas.mjs [--container NAME] [--hours N] [--permissions dlrw]`); + process.exit(1); + } + args[field] = argv[i + 1]; +} + +const account = process.env.STORAGE_ACCOUNT_NAME; +const key = process.env.STORAGE_ACCOUNT_KEY; +if (!account || !key) { + console.error('error: STORAGE_ACCOUNT_NAME and STORAGE_ACCOUNT_KEY must be set'); + process.exit(1); +} + +const now = new Date(); +const sas = generateBlobSASQueryParameters( + { + containerName: args.container, + permissions: ContainerSASPermissions.parse(args.permissions), + // Five minutes of clock-skew allowance so a fast runner does not present a + // token whose start time the storage service still considers in the future. + startsOn: new Date(now.getTime() - 5 * 60 * 1000), + expiresOn: new Date(now.getTime() + Number(args.hours) * 60 * 60 * 1000), + }, + new StorageSharedKeyCredential(account, key), +); + +process.stdout.write(sas.toString()); diff --git a/scripts/set-blob-metadata.mjs b/scripts/set-blob-metadata.mjs new file mode 100644 index 0000000000..b142f06971 --- /dev/null +++ b/scripts/set-blob-metadata.mjs @@ -0,0 +1,264 @@ +#!/usr/bin/env node + +/** + * Stamp `public_url` metadata and content types onto the static-site container. + * + * Usage: + * node scripts/set-blob-metadata.mjs \ + * --account NAME --key KEY --container '$web' --base-url https://docs.example.com + * + * Environment fallbacks: STORAGE_ACCOUNT_NAME, STORAGE_ACCOUNT_KEY, APP_EXTERNAL_URL + * Pass --force to rewrite every blob regardless of its current state. + * Pass --dry-run to list and plan without writing anything. + * + * Replaces two deploy steps that each walked the whole container: a per-blob + * `az storage blob metadata update` loop (~0.6s of CLI cold start each, 20 at a + * time, ~47 minutes for the site's ~86k blobs) and 19 `az storage blob + * update-batch` calls, one per file extension. Both are folded into a single + * listing pass here: `includeMetadata` returns each blob's metadata and + * properties inline, so deciding what needs writing costs no extra round trips, + * and only blobs that are actually missing or stale get a PUT. + */ + +import { BlobServiceClient, StorageSharedKeyCredential } from '@azure/storage-blob'; +import { pathToFileURL } from 'url'; + +const METADATA_KEY = 'public_url'; + +// Mirrors the extension/content-type pairs the previous `az storage blob +// update-batch` steps applied. Extensions absent from this map keep whatever +// content type azcopy inferred at upload, which is what update-batch did too. +const CONTENT_TYPE_BY_EXTENSION = new Map(Object.entries({ + css: 'text/css', + js: 'application/javascript', + mjs: 'application/javascript', + json: 'application/json', + html: 'text/html', + htm: 'text/html', + xml: 'application/xml', + txt: 'text/plain', + png: 'image/png', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + gif: 'image/gif', + webp: 'image/webp', + svg: 'image/svg+xml', + ico: 'image/x-icon', + woff: 'font/woff', + woff2: 'font/woff2', + ttf: 'font/ttf', + otf: 'font/otf', +})); + +// Azure sends metadata as `x-ms-meta-*` headers, which are latin-1 on the wire, +// so a raw blob name is not safe to interpolate: the site ships names carrying +// a Cyrillic homoglyph and a curly quote, and both throw at socket-write time. +// Percent-encode each path segment and leave the separators intact. +export function publicUrlFor(baseUrl, blobName) { + const encoded = blobName.split('/').map(encodeURIComponent).join('/'); + return `${baseUrl.replace(/\/+$/, '')}/${encoded}`; +} + +export function contentTypeFor(blobName) { + const base = blobName.slice(blobName.lastIndexOf('/') + 1); + const dot = base.lastIndexOf('.'); + if (dot <= 0) return undefined; + return CONTENT_TYPE_BY_EXTENSION.get(base.slice(dot + 1).toLowerCase()); +} + +/** + * Decide what a single blob needs. Returns null when it is already correct, + * so an unchanged deploy writes nothing. + */ +export function planFor(blob, baseUrl, force = false) { + const wantUrl = publicUrlFor(baseUrl, blob.name); + const wantType = contentTypeFor(blob.name); + const properties = blob.properties ?? {}; + + const needsMetadata = force || blob.metadata?.[METADATA_KEY] !== wantUrl; + const needsContentType = + wantType !== undefined && (force || properties.contentType !== wantType); + + if (!needsMetadata && !needsContentType) return null; + return { name: blob.name, wantUrl, wantType, needsMetadata, needsContentType, properties }; +} + +/** Run `tasks` with at most `limit` in flight, collecting rejections per task. */ +async function runPool(tasks, limit, onSettled) { + let next = 0; + const workers = Array.from({ length: Math.min(limit, tasks.length) }, async () => { + while (next < tasks.length) { + const index = next++; + try { + await tasks[index](); + onSettled(null); + } catch (error) { + onSettled(`${tasks[index].blobName}: ${error?.message ?? error}`); + } + } + }); + await Promise.all(workers); +} + +/** A bad invocation, reported as a one-line message rather than a stack trace. */ +class UsageError extends Error {} + +function parseArgs(argv) { + const args = { + account: process.env.STORAGE_ACCOUNT_NAME, + key: process.env.STORAGE_ACCOUNT_KEY, + container: '$web', + baseUrl: process.env.APP_EXTERNAL_URL ?? '', + force: false, + dryRun: false, + concurrency: 64, + endpoint: process.env.AZURE_BLOB_ENDPOINT, + }; + const flags = new Map([ + ['--account', 'account'], + ['--key', 'key'], + ['--container', 'container'], + ['--base-url', 'baseUrl'], + ['--concurrency', 'concurrency'], + ['--endpoint', 'endpoint'], + ]); + + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--force') { + args.force = true; + continue; + } + if (argv[i] === '--dry-run') { + args.dryRun = true; + continue; + } + const field = flags.get(argv[i]); + if (!field) throw new UsageError(`unknown argument: ${argv[i]}`); + if (i + 1 >= argv.length) throw new UsageError(`${argv[i]} requires a value`); + args[field] = argv[++i]; + } + + args.concurrency = Number(args.concurrency); + const missing = ['account', 'key', 'baseUrl'].filter((field) => !args[field]); + if (missing.length) throw new UsageError(`missing required argument(s): ${missing.join(', ')}`); + if (!Number.isInteger(args.concurrency) || args.concurrency < 1) { + throw new UsageError('--concurrency must be a positive integer'); + } + return args; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + // --endpoint exists so the script can be pointed at the Azurite emulator. + const client = new BlobServiceClient( + args.endpoint ?? `https://${args.account}.blob.core.windows.net`, + new StorageSharedKeyCredential(args.account, args.key), + // The SDK's storage retry policy handles throttling (503) and transient + // resets with Azure-aware backoff, so the pool below does not retry itself. + { retryOptions: { maxTries: 5 } }, + ); + const container = client.getContainerClient(args.container); + + console.log(`Listing blobs in '${args.container}'...`); + let listed = 0; + const plans = []; + for await (const blob of container.listBlobsFlat({ includeMetadata: true })) { + listed++; + const plan = planFor(blob, args.baseUrl, args.force); + if (plan) plans.push(plan); + } + + const metadataWrites = plans.filter((plan) => plan.needsMetadata).length; + const contentTypeWrites = plans.filter((plan) => plan.needsContentType).length; + console.log( + ` ${listed} blobs listed, ${plans.length} need writes ` + + `(${metadataWrites} metadata, ${contentTypeWrites} content type)`, + ); + + if (!plans.length) { + console.log('All blobs already carry the correct metadata and content types.'); + return 0; + } + + if (args.dryRun) { + console.log('Dry run: sample of blobs that would be written:'); + for (const plan of plans.slice(0, 20)) { + const what = [plan.needsMetadata && 'metadata', plan.needsContentType && `type=${plan.wantType}`] + .filter(Boolean) + .join(', '); + console.log(` ${plan.name} (${what})`); + } + return 0; + } + + const tasks = plans.map((plan) => { + const task = async () => { + const blobClient = container.getBlobClient(plan.name); + if (plan.needsMetadata) { + // setMetadata replaces the whole metadata dict, matching the semantics + // of `az storage blob metadata update --metadata public_url=...`. + await blobClient.setMetadata({ [METADATA_KEY]: plan.wantUrl }); + } + if (plan.needsContentType) { + // setHTTPHeaders replaces every header it accepts, so carry the other + // content settings through rather than blanking them. + await blobClient.setHTTPHeaders({ + blobContentType: plan.wantType, + blobCacheControl: plan.properties.cacheControl, + blobContentEncoding: plan.properties.contentEncoding, + blobContentLanguage: plan.properties.contentLanguage, + blobContentDisposition: plan.properties.contentDisposition, + }); + } + }; + task.blobName = plan.name; + return task; + }); + + const started = Date.now(); + const errors = []; + let done = 0; + let failed = 0; + + await runPool(tasks, args.concurrency, (error) => { + if (error) { + failed++; + if (errors.length < 20) errors.push(error); + } else { + done++; + } + const processed = done + failed; + if (processed % 5000 === 0) { + const rate = processed / Math.max((Date.now() - started) / 1000, 1e-9); + console.log(` ${processed}/${tasks.length} (${rate.toFixed(0)}/s)`); + } + }); + + const elapsed = ((Date.now() - started) / 1000).toFixed(1); + console.log(`Updated ${done} blobs in ${elapsed}s (${failed} failed)`); + + if (errors.length) { + console.error('First failures:'); + for (const error of errors) console.error(` ${error}`); + } + return failed ? 1 : 0; +} + +// Only run when invoked directly, so the helpers above stay importable by tests. +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main() + .then((code) => process.exit(code)) + .catch((error) => { + if (error instanceof UsageError) { + console.error(`error: ${error.message}`); + console.error( + 'usage: node scripts/set-blob-metadata.mjs --account NAME --key KEY ' + + "[--container '$web'] --base-url URL [--force] [--dry-run] [--concurrency N] [--endpoint URL]", + ); + } else { + console.error(error); + } + process.exit(1); + }); +} diff --git a/scripts/test-set-blob-metadata-integration.mjs b/scripts/test-set-blob-metadata-integration.mjs new file mode 100644 index 0000000000..800ffd74c8 --- /dev/null +++ b/scripts/test-set-blob-metadata-integration.mjs @@ -0,0 +1,103 @@ +#!/usr/bin/env node + +/** + * Integration test for scripts/set-blob-metadata.mjs and + * scripts/mint-container-sas.mjs against the Azurite blob emulator. + * + * Usage: + * npx --package azurite azurite-blob --silent --location /tmp/azurite & + * node scripts/test-set-blob-metadata-integration.mjs + * + * Skips (exit 0) when nothing is listening on the emulator port, so it is + * safe to include in a suite that runs without Azurite. + */ + +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { BlobServiceClient, ContainerClient, StorageSharedKeyCredential } from '@azure/storage-blob'; + +const ACCOUNT = 'devstoreaccount1'; +const KEY = 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=='; +const ENDPOINT = `http://127.0.0.1:10000/${ACCOUNT}`; +const BASE = 'https://docs.example.com'; +const CONTAINER = '$web'; +const probe = await fetch(`${ENDPOINT}/?comp=list`).catch(() => null); +if (!probe) { + console.log(`Azurite not reachable at ${ENDPOINT}, skipping integration test`); + process.exit(0); +} + +const env = { ...process.env, STORAGE_ACCOUNT_NAME: ACCOUNT, STORAGE_ACCOUNT_KEY: KEY }; + +const cred = new StorageSharedKeyCredential(ACCOUNT, KEY); +const service = new BlobServiceClient(ENDPOINT, cred); +const container = service.getContainerClient(CONTAINER); +await container.deleteIfExists(); +await container.create(); + +const cyrillic = 'docs/kb/how_to_trigger_a_workflow_when_a_user_сreates_a_group/index.html'; +const seed = [ + ['index.html', 'text/plain', undefined], + [cyrillic, 'text/html', undefined], + ['assets/images/ADMX Template-480a6d.png', 'application/octet-stream', undefined], + ['a.pdf', 'application/pdf', undefined], + ['x.webp', 'image/webp', undefined], + ['ok.css', 'text/css', `${BASE}/ok.css`], + ['stale.js', 'application/javascript', 'https://old.example.com/stale.js'], +]; +for (const [name, type, url] of seed) { + await container.getBlockBlobClient(name).upload('x', 1, { + blobHTTPHeaders: { blobContentType: type, blobCacheControl: 'max-age=600' }, + metadata: url ? { public_url: url } : undefined, + }); +} + +const run = (...extra) => + execFileSync('node', ['scripts/set-blob-metadata.mjs', '--container', CONTAINER, '--base-url', BASE, '--endpoint', ENDPOINT, ...extra], { env, encoding: 'utf8' }); + +let out = run('--dry-run'); +console.log(out); +assert.match(out, /7 blobs listed, 6 need writes \(6 metadata, 2 content type\)/); +assert.equal((await container.getBlobClient('index.html').getProperties()).contentType, 'text/plain', 'dry run must not write'); + +out = run(); +console.log(out); +assert.match(out, /Updated 6 blobs in [\d.]+s \(0 failed\)/); + +const expect = { + 'index.html': 'text/html', + [cyrillic]: 'text/html', + 'assets/images/ADMX Template-480a6d.png': 'image/png', + 'a.pdf': 'application/pdf', + 'x.webp': 'image/webp', + 'ok.css': 'text/css', + 'stale.js': 'application/javascript', +}; +for (const [name, type] of Object.entries(expect)) { + const p = await container.getBlobClient(name).getProperties(); + assert.equal(p.contentType, type, `${name} content type`); + assert.equal(p.cacheControl, 'max-age=600', `${name} cache-control preserved`); + assert.equal(p.metadata.public_url, name.split('/').map(encodeURIComponent).join('/').replace(/^/, `${BASE}/`), `${name} public_url`); +} + +out = run(); +console.log(out); +assert.match(out, /7 blobs listed, 0 need writes/); +assert.match(out, /All blobs already carry/); + +out = run('--force', '--dry-run'); +assert.match(out, /7 need writes \(7 metadata, 6 content type\)/); + +// SAS: mint with the script, then use it (no account key) to list and write. +const sas = execFileSync('node', ['scripts/mint-container-sas.mjs', '--container', CONTAINER, '--hours', '1', '--permissions', 'dlrw'], { env, encoding: 'utf8' }); +assert.match(sas, /sp=rwdl/); +const viaSas = new ContainerClient(`${ENDPOINT}/${CONTAINER}?${sas}`); +let n = 0; +for await (const _ of viaSas.listBlobsFlat()) n++; +assert.equal(n, 7, 'SAS can list'); +await viaSas.getBlockBlobClient('sas-written.txt').upload('y', 1); +await viaSas.deleteBlob('sas-written.txt'); +const readOnly = execFileSync('node', ['scripts/mint-container-sas.mjs', '--container', CONTAINER, '--permissions', 'l'], { env, encoding: 'utf8' }); +await assert.rejects(new ContainerClient(`${ENDPOINT}/${CONTAINER}?${readOnly}`).getBlockBlobClient('nope.txt').upload('y', 1), (e) => e.statusCode === 403); + +console.log('INTEGRATION: all assertions passed'); diff --git a/scripts/test-set-blob-metadata.mjs b/scripts/test-set-blob-metadata.mjs new file mode 100644 index 0000000000..dbb0bf882c --- /dev/null +++ b/scripts/test-set-blob-metadata.mjs @@ -0,0 +1,175 @@ +#!/usr/bin/env node + +/** + * Tests for scripts/set-blob-metadata.mjs + * + * Usage: + * node --test scripts/test-set-blob-metadata.mjs + * + * Covers the pure decision logic only: URL construction, content-type lookup, + * and the skip/write plan. The Azure calls themselves are exercised on deploy. + */ + +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { contentTypeFor, planFor, publicUrlFor } from './set-blob-metadata.mjs'; + +const BASE = 'https://docs.example.com'; + +test('publicUrlFor keeps path separators and joins to the base', () => { + assert.equal( + publicUrlFor(BASE, 'docs/pingcastle/index.html'), + 'https://docs.example.com/docs/pingcastle/index.html', + ); +}); + +test('publicUrlFor strips trailing slashes from the base', () => { + assert.equal(publicUrlFor(`${BASE}/`, 'a.html'), 'https://docs.example.com/a.html'); + assert.equal(publicUrlFor(`${BASE}///`, 'a.html'), 'https://docs.example.com/a.html'); +}); + +// These two names are real files in the built site and are exactly what broke +// the previous implementation: it interpolated them raw into an x-ms-meta-* +// header, which throws UnicodeEncodeError at socket-write time. +test('publicUrlFor encodes non-ASCII blob names to pure ASCII', () => { + const cyrillic = 'docs/kb/how_to_trigger_a_workflow_when_a_user_сreates_a_group/index.html'; + const curlyQuote = 'docs/kb/error_“collect_information/index.html'; + + for (const name of [cyrillic, curlyQuote]) { + const url = publicUrlFor(BASE, name); + assert.doesNotThrow(() => Buffer.from(url, 'latin1')); + // eslint-disable-next-line no-control-regex + assert.match(url, /^[\x00-\x7F]*$/, `expected ASCII-only output, got ${url}`); + } + + assert.equal( + publicUrlFor(BASE, cyrillic), + 'https://docs.example.com/docs/kb/how_to_trigger_a_workflow_when_a_user_%D1%81reates_a_group/index.html', + ); +}); + +test('publicUrlFor encodes spaces and URL-significant characters', () => { + assert.equal( + publicUrlFor(BASE, 'assets/images/ADMX Template-480a6d.png'), + 'https://docs.example.com/assets/images/ADMX%20Template-480a6d.png', + ); + // A literal '?' or '#' in a name would otherwise truncate the URL. + assert.equal(publicUrlFor(BASE, 'a/b?c#d.html'), 'https://docs.example.com/a/b%3Fc%23d.html'); +}); + +test('contentTypeFor maps the extensions the az update-batch steps covered', () => { + const expected = { + 'a.css': 'text/css', + 'a.js': 'application/javascript', + 'a.mjs': 'application/javascript', + 'a.json': 'application/json', + 'a.html': 'text/html', + 'a.htm': 'text/html', + 'a.xml': 'application/xml', + 'a.txt': 'text/plain', + 'a.png': 'image/png', + 'a.jpg': 'image/jpeg', + 'a.jpeg': 'image/jpeg', + 'a.gif': 'image/gif', + 'a.webp': 'image/webp', + 'a.svg': 'image/svg+xml', + 'a.ico': 'image/x-icon', + 'a.woff': 'font/woff', + 'a.woff2': 'font/woff2', + 'a.ttf': 'font/ttf', + 'a.otf': 'font/otf', + }; + for (const [name, type] of Object.entries(expected)) { + assert.equal(contentTypeFor(name), type, `${name} should map to ${type}`); + } + assert.equal(Object.keys(expected).length, 19, 'should cover all 19 original patterns'); +}); + +test('contentTypeFor is case-insensitive and ignores unknown or absent extensions', () => { + assert.equal(contentTypeFor('a/B.WEBP'), 'image/webp'); + assert.equal(contentTypeFor('a/README'), undefined); + assert.equal(contentTypeFor('a/archive.tar.gz'), undefined); + // A dotfile is a name, not an extension. + assert.equal(contentTypeFor('a/.nojekyll'), undefined); + // A dot in a directory must not be read as the file's extension. + assert.equal(contentTypeFor('docs/accessanalyzer/12.0/index'), undefined); +}); + +test('planFor returns null when metadata and content type are both current', () => { + const blob = { + name: 'a.html', + metadata: { public_url: `${BASE}/a.html` }, + properties: { contentType: 'text/html' }, + }; + assert.equal(planFor(blob, BASE), null); +}); + +test('planFor flags a missing or stale public_url', () => { + const properties = { contentType: 'text/html' }; + + const missing = planFor({ name: 'a.html', metadata: {}, properties }, BASE); + assert.equal(missing.needsMetadata, true); + assert.equal(missing.needsContentType, false); + + const stale = planFor( + { name: 'a.html', metadata: { public_url: 'https://old.example.com/a.html' }, properties }, + BASE, + ); + assert.equal(stale.needsMetadata, true); + assert.equal(stale.wantUrl, `${BASE}/a.html`); +}); + +test('planFor flags a wrong content type independently of metadata', () => { + const plan = planFor( + { + name: 'a.webp', + metadata: { public_url: `${BASE}/a.webp` }, + properties: { contentType: 'application/octet-stream' }, + }, + BASE, + ); + assert.equal(plan.needsMetadata, false); + assert.equal(plan.needsContentType, true); + assert.equal(plan.wantType, 'image/webp'); +}); + +test('planFor leaves unmapped extensions alone rather than blanking them', () => { + const plan = planFor( + { + name: 'a.pdf', + metadata: { public_url: `${BASE}/a.pdf` }, + properties: { contentType: 'application/pdf' }, + }, + BASE, + ); + assert.equal(plan, null); +}); + +test('planFor with --force rewrites even a fully current blob', () => { + const blob = { + name: 'a.html', + metadata: { public_url: `${BASE}/a.html` }, + properties: { contentType: 'text/html' }, + }; + const plan = planFor(blob, BASE, true); + assert.equal(plan.needsMetadata, true); + assert.equal(plan.needsContentType, true); +}); + +test('planFor carries other content settings through for setHTTPHeaders', () => { + const plan = planFor( + { + name: 'a.html', + metadata: {}, + properties: { contentType: 'text/plain', cacheControl: 'max-age=600' }, + }, + BASE, + ); + assert.equal(plan.properties.cacheControl, 'max-age=600'); +}); + +test('planFor tolerates a blob listed without metadata', () => { + const plan = planFor({ name: 'a.html', properties: { contentType: 'text/html' } }, BASE); + assert.equal(plan.needsMetadata, true); +});