build: content-hashed asset copies + asset-manifest.json

README.md

@@ -94,6 +94,25 @@ and run `$ npm run dist`. The files will be created under `dist/`.
 Because Esplora is a single-page app, the HTTP server needs to be configured to serve the `index.html` file in reply to missing pages.
 See [`contrib/nginx.conf.in`](contrib/nginx.conf.in) for example nginx configuration (TL;DR: `try_files $uri /index.html`).
 
+### Content-hashed frontend assets
+
+Production builds create content-hashed copies of the app, stylesheets, lazy-loaded
+scripts and fonts referenced directly by the stylesheets by default. Original
+filenames are retained for rollback and cached HTML. `asset-manifest.json` in the
+output directory (`dist/`, or `DEST`) maps original relative paths to hashed paths.
+Set `HASH_ASSETS=0` to disable hashing and use plain filenames without a manifest.
+
+Both server-side renderers accept `ASSET_MANIFEST` pointing to this file (default:
+`DEST/asset-manifest.json`, or the project's `dist/asset-manifest.json`). Missing
+manifests fall back to plain filenames; `STATIC_ROOT` is applied to either form.
+
+Deployment must upload new hashed files with
+`Cache-Control: public, max-age=31536000, immutable` **before** updating HTML, and
+must not delete old hashed files immediately after deployment (cached HTML and
+open tabs still reference them). If the server falls back to `index.html` for
+unknown paths, hashed-asset misses must return a real 404 instead, so a missing
+file is never cached as HTML.
+
 ## Pre-rendering server (noscript)
 
 To start a pre-rendering server that generates static HTML replies suitable for noscript users, run:

build.sh

@@ -31,9 +31,6 @@ cp -RL www/* $CUSTOM_ASSETS $DEST/
 # Assemble base CSS modules followed by flavor customizations
 node scripts/assemble-css.js "$DEST/style.css" $CUSTOM_CSS
 
-# Index HTML
-pug client/index.pug -o $DEST
-
 # Open search (requires absolute CANONICAL_URL)
 if [ -n "$CANONICAL_URL" ]; then
   pug client/opensearch.pug -E xml -o $DEST
@@ -42,11 +39,26 @@ fi
 # RTLify CSS
 cat $DEST/style.css | node -p "require('cssjanus').transform(fs.readFileSync('/dev/stdin').toString(), false, true)" > $DEST/style-rtl.css
 
+# Resolve lazy-loaded scripts before envify bakes their filenames into app.js.
+export ASSET_MANIFEST="$DEST/asset-manifest.json"
+unset INSTASCAN_ASSET INFINITE_SCROLL_ASSET
+if [[ "$HASH_ASSETS" != "0" ]]; then
+  node scripts/hash-assets.js "$DEST" --scripts-only
+  export INSTASCAN_ASSET=$(node -p "require('./scripts/asset-manifest').loadAssetManifest()['instascan.min.js']")
+  export INFINITE_SCROLL_ASSET=$(node -p "require('./scripts/asset-manifest').loadAssetManifest()['js/infinite-scroll.js']")
+fi
+
 # Browserify bundle
 # --no-dedupe needed due to https://github.com/substack/bundle-collapser/issues/20 https://github.com/browserify/browserify/issues/1450
 (cd client && browserify --no-dedupe -p bundle-collapser/plugin src/run-browser.js \
   | ( [[ "$NODE_ENV" != "development" ]] && uglifyjs -cm || cat ) ) \
   > $DEST/app.js
 
+# Fonts are rewritten before CSS hashing; HTML is rendered only after app.js is hashed.
+if [[ "$HASH_ASSETS" != "0" ]]; then
+  node scripts/hash-assets.js "$DEST"
+fi
+pug client/index.pug -o $DEST -O "$(node scripts/asset-manifest.js)"
+
 # Pre-render notfound.html
 babel-node render-view.js '{"view":"error","error":"Page Not Found"}' > $DEST/notfound.html

client/index.pug

@@ -4,7 +4,10 @@ doctype html
 - staticRoot = process.env.STATIC_ROOT || ''
 - pageTitle = prerender_title ? `${prerender_title} · ${baseTitle}` : baseTitle
 
-- cssPath = t ? t`style.css` : 'style.css'
+- assets = typeof assetManifest === 'object' && assetManifest || {}
+- ltrPath = staticRoot + (assets['style.css'] || 'style.css')
+- rtlPath = staticRoot + (assets['style-rtl.css'] || 'style-rtl.css')
+- cssPath = t && t`style.css` === 'style-rtl.css' ? rtlPath : ltrPath
 - lang = t ? t.lang_id : 'en'
 - dir = t ? t`ltr` : 'ltr'
 
@@ -29,7 +32,7 @@ head
 
   meta(name='viewport', content='width=device-width, initial-scale=1')
   link(rel='shortcut icon', type='image/png', href=staticRoot+'img/favicon.png')
-  link(rel='stylesheet', href=staticRoot+cssPath)
+  link(rel='stylesheet', href=cssPath, data-ltr=ltrPath, data-rtl=rtlPath)
 
   if process.env.CANONICAL_URL
     //- open search requires the absolute URL of the explorer
@@ -53,5 +56,5 @@ body(lang=lang, dir=dir)
         .nojs-shim
           p This block explorer needs JavaScript to run. Please enable it and reload the page.
 
-    script(src=staticRoot+'app.js', async)
+    script(src=staticRoot+(assets['app.js'] || 'app.js'), async)
   != process.env.FOOT_HTML

client/src/app.js

@@ -799,9 +799,14 @@ export default function main(
     // Click-to-copy
     if (navigator.clipboard) copy$.subscribe(text => navigator.clipboard.writeText(text))
 
-    // Switch stylesheet based on current language
-    const stylesheet = document.querySelector('link[href="style.css"]')
-    t$.map(t => t`style.css`).distinctUntilChanged().subscribe(styleSrc =>
+    // Keep the existing translation's RTL marker, but resolve URLs from the rendered link.
+    const stylesheet = document.querySelector('link[rel="stylesheet"][data-ltr]')
+      || document.querySelector('link[rel="stylesheet"]') // HTML cached before hashing was introduced
+    t$.map(t => {
+      const rtl = t`style.css` === 'style-rtl.css'
+      return stylesheet.getAttribute(rtl ? 'data-rtl' : 'data-ltr')
+        || (process.env.STATIC_ROOT || '') + (rtl ? 'style-rtl.css' : 'style.css')
+    }).distinctUntilChanged().subscribe(styleSrc =>
       stylesheet.getAttribute('href') != styleSrc && (stylesheet.href = styleSrc))
 
     // Apply language and text direction to root element

client/src/driver/instascan.js

@@ -1,6 +1,7 @@
 import { Observable as O } from '../rxjs'
 
 const staticRoot = process.env.STATIC_ROOT || ''
+    , instascanAsset = process.env.INSTASCAN_ASSET || 'instascan.min.js'
 
 // check for WebRTC camera support
 if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
@@ -11,11 +12,11 @@ if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
     if (loaded) return;
     loaded = true;
     const script = document.createElement('script')
-    script.src = `${staticRoot}instascan.min.js`
+    script.src = `${staticRoot}${instascanAsset}`
     document.body.appendChild(script)
   }
 
-  const Instascan$ = O.fromEvent(document.body, 'load', true).filter(e => e.target?.src?.endsWith('/instascan.min.js')).map(_ => window.Instascan).share()
+  const Instascan$ = O.fromEvent(document.body, 'load', true).filter(e => e.target?.src?.endsWith(`/${instascanAsset}`)).map(_ => window.Instascan).share()
       , Scanner$   = Instascan$.map(Instascan => Instascan.Scanner)
       , Camera$    = Instascan$.map(Instascan => Instascan.Camera)
 

client/src/views/lander.js

@@ -16,7 +16,8 @@ const LandingPage = ({ t, ...S }) => {
     const logos = logoSources
     return  layout(
     <div className="landing-page">
-        <script src={`${staticRoot}/js/infinite-scroll.js`} async></script>
+        {/* Read at render time: the prerender server sets it from the manifest at startup. */}
+        <script src={`${staticRoot}${process.env.INFINITE_SCROLL_ASSET || 'js/infinite-scroll.js'}`} async></script>
         <div className="blur-orange"></div>
         <div className="blur-green"></div>
         <div className="laser-lines"></div>

contrib/runits/prerenderer.runit

@@ -6,6 +6,8 @@ export NODE_ENV=${NODE_ENV:-production}
 export API_URL=http://127.0.0.1/{NGINX_PATH}api
 export TZ=UTC
 export SOCKET_PATH=/var/prerender-http.sock
+# Content-hashed asset names for this flavor's build (see README).
+export ASSET_MANIFEST={STATIC_DIR}/asset-manifest.json
 
 exec 2>&1
 exec /bin/bash -c 'source /root/.nvm/nvm.sh && cd /srv/explorer/source && npm run prerender-server -- --cluster {FLAVOR}'

prerender-server/src/server.js

@@ -7,6 +7,11 @@ import promClient from 'prom-client'
 
 import l10n from '../client/l10n'
 import render from '../client/run-server'
+import { loadAssetManifest, exportAssetEnv } from '../../scripts/asset-manifest'
+
+// Load the flavor's manifest once; views read the lazy-script names from env at render time.
+const assetManifest = loadAssetManifest()
+exportAssetEnv(assetManifest)
 
 if (!process.env.API_URL) {
   throw new Error('API_URL environment variable is required but not defined.');
@@ -136,7 +141,8 @@ function doRender(req, res, next) {
 
     res.status(resp.status || 200)
     res.render(indexView, {
-      prerender_title: resp.title
+      assetManifest
+      , prerender_title: resp.title
       , prerender_html: resp.html
       , canon_url: canonBase ? canonBase + req.url : null
       , noscript: true

render-view.js

@@ -9,7 +9,8 @@ state.t = l10n[state.lang || 'en']
 state.page = { pathname: '', query: {} }
 
 require('pug').renderFile('client/index.pug', {
-  prerender_html: require('snabbdom-to-html')(view(state))
+  assetManifest: require('./scripts/asset-manifest').loadAssetManifest()
+  , prerender_html: require('snabbdom-to-html')(view(state))
 }, (err, html) => {
   if (err) throw err
   console.log(html)

scripts/asset-manifest.js

@@ -0,0 +1,35 @@
+const fs = require('fs')
+const path = require('path')
+
+// Shared by the static build and both server-side renderers. Dev needs no manifest.
+function loadAssetManifest() {
+  if (process.env.HASH_ASSETS === '0') return {}
+  const filename = process.env.ASSET_MANIFEST
+    || path.resolve(__dirname, '..', process.env.DEST || 'dist', 'asset-manifest.json')
+  try {
+    return JSON.parse(fs.readFileSync(filename, 'utf8'))
+  } catch (error) {
+    if (error.code === 'ENOENT') return {}
+    throw error
+  }
+}
+
+// Lazy-loaded scripts are referenced from views through env vars. The browser bundle gets
+// them from envify at build time; server-side renderers set them from the manifest.
+const lazyAssetEnv = {
+  INSTASCAN_ASSET: 'instascan.min.js',
+  INFINITE_SCROLL_ASSET: 'js/infinite-scroll.js',
+}
+
+function exportAssetEnv(manifest, env=process.env) {
+  for (const [ name, original ] of Object.entries(lazyAssetEnv)) {
+    if (!env[name] && manifest[original]) env[name] = manifest[original]
+  }
+  return env
+}
+
+module.exports = { loadAssetManifest, exportAssetEnv }
+
+if (require.main === module) {
+  console.log(JSON.stringify({ assetManifest: loadAssetManifest() }))
+}

scripts/hash-assets.js

@@ -0,0 +1,65 @@
+const fs = require('fs')
+const path = require('path')
+const { createHash } = require('crypto')
+
+const lazyScripts = [ 'instascan.min.js', 'js/infinite-scroll.js' ]
+const stylesheets = [ 'style.css', 'style-rtl.css' ]
+
+function hashAssets(destination, scriptsOnly=false) {
+  const root = path.resolve(destination)
+      , manifestPath = path.join(root, 'asset-manifest.json')
+      , previous = fs.existsSync(manifestPath) ? JSON.parse(fs.readFileSync(manifestPath, 'utf8')) : {}
+      , originals = Object.fromEntries(Object.entries(previous).map(([ original, hashed ]) => [ hashed, original ]))
+      , manifest = {}
+
+  function copyHashed(relative) {
+    if (manifest[relative]) return manifest[relative]
+    const content = fs.readFileSync(path.join(root, relative))
+        , hash = createHash('sha256').update(content).digest('hex').slice(0, 12)
+        , extension = path.extname(relative)
+        , hashed = `${relative.slice(0, -extension.length)}.${hash}${extension}`
+    fs.writeFileSync(path.join(root, hashed), content)
+    manifest[relative] = hashed
+    return hashed
+  }
+
+  lazyScripts.forEach(copyHashed)
+
+  if (!scriptsOnly) {
+    for (const stylesheet of stylesheets) {
+      const filename = path.join(root, stylesheet)
+          , css = fs.readFileSync(filename, 'utf8')
+          , rewritten = css.replace(/url\(\s*(['"]?)([^'"\)]+?)\1\s*\)/gi, (match, quote, url) => {
+            // Only local font URLs belong to phase 1. Preserve query strings/fragments.
+            if (/^(?:[a-z][a-z\d+.-]*:|\/\/)/i.test(url)) return match
+            const [ , pathname, suffix ] = url.match(/^([^?#]+)([?#].*)?$/) || []
+            if (!pathname || !/\.(woff2?|ttf)$/i.test(pathname)) return match
+            const absolute = pathname.startsWith('/')
+                , relative = path.posix.normalize(absolute ? pathname.slice(1) : path.posix.join(path.posix.dirname(stylesheet), pathname))
+            if (relative.startsWith('../')) throw new Error(`Font URL escapes output directory: ${url}`)
+            // Re-running the script must not hash an already hashed filename again.
+            const original = originals[relative] || relative
+                , hashed = copyHashed(original)
+                , rewrittenUrl = absolute ? '/' + hashed : path.posix.relative(path.posix.dirname(stylesheet), hashed)
+            return `url(${quote}${rewrittenUrl}${suffix || ''}${quote})`
+          })
+      fs.writeFileSync(filename, rewritten)
+      copyHashed(stylesheet)
+    }
+    copyHashed('app.js')
+  }
+
+  const sorted = Object.fromEntries(Object.entries(manifest).sort(([ a ], [ b ]) => a.localeCompare(b, 'en')))
+  fs.writeFileSync(manifestPath, JSON.stringify(sorted, null, 2) + '\n')
+  return sorted
+}
+
+module.exports = { hashAssets }
+
+if (require.main === module && process.env.HASH_ASSETS !== '0') {
+  const [ destination, mode ] = process.argv.slice(2)
+  if (!destination || (mode && mode !== '--scripts-only')) {
+    throw new Error('Usage: node scripts/hash-assets.js <output-directory> [--scripts-only]')
+  }
+  hashAssets(destination, mode === '--scripts-only')
+}