diff --git a/README.md b/README.md index 8ad9790f..e9f2a020 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,25 @@ and run `$ npm run dist`. The files will be created under `dist/`. Because Esplora is a single-page app, the HTTP server needs to be configured to serve the `index.html` file in reply to missing pages. See [`contrib/nginx.conf.in`](contrib/nginx.conf.in) for example nginx configuration (TL;DR: `try_files $uri /index.html`). +### Content-hashed frontend assets + +Production builds create content-hashed copies of the app, stylesheets, lazy-loaded +scripts and fonts referenced directly by the stylesheets by default. Original +filenames are retained for rollback and cached HTML. `asset-manifest.json` in the +output directory (`dist/`, or `DEST`) maps original relative paths to hashed paths. +Set `HASH_ASSETS=0` to disable hashing and use plain filenames without a manifest. + +Both server-side renderers accept `ASSET_MANIFEST` pointing to this file (default: +`DEST/asset-manifest.json`, or the project's `dist/asset-manifest.json`). Missing +manifests fall back to plain filenames; `STATIC_ROOT` is applied to either form. + +Deployment must upload new hashed files with +`Cache-Control: public, max-age=31536000, immutable` **before** updating HTML, and +must not delete old hashed files immediately after deployment (cached HTML and +open tabs still reference them). If the server falls back to `index.html` for +unknown paths, hashed-asset misses must return a real 404 instead, so a missing +file is never cached as HTML. + ## Pre-rendering server (noscript) To start a pre-rendering server that generates static HTML replies suitable for noscript users, run: diff --git a/build.sh b/build.sh index 3bb37a98..fb9d671d 100755 --- a/build.sh +++ b/build.sh @@ -31,9 +31,6 @@ cp -RL www/* $CUSTOM_ASSETS $DEST/ # Assemble base CSS modules followed by flavor customizations node scripts/assemble-css.js "$DEST/style.css" $CUSTOM_CSS -# Index HTML -pug client/index.pug -o $DEST - # Open search (requires absolute CANONICAL_URL) if [ -n "$CANONICAL_URL" ]; then pug client/opensearch.pug -E xml -o $DEST @@ -42,11 +39,26 @@ fi # RTLify CSS cat $DEST/style.css | node -p "require('cssjanus').transform(fs.readFileSync('/dev/stdin').toString(), false, true)" > $DEST/style-rtl.css +# Resolve lazy-loaded scripts before envify bakes their filenames into app.js. +export ASSET_MANIFEST="$DEST/asset-manifest.json" +unset INSTASCAN_ASSET INFINITE_SCROLL_ASSET +if [[ "$HASH_ASSETS" != "0" ]]; then + node scripts/hash-assets.js "$DEST" --scripts-only + export INSTASCAN_ASSET=$(node -p "require('./scripts/asset-manifest').loadAssetManifest()['instascan.min.js']") + export INFINITE_SCROLL_ASSET=$(node -p "require('./scripts/asset-manifest').loadAssetManifest()['js/infinite-scroll.js']") +fi + # Browserify bundle # --no-dedupe needed due to https://github.com/substack/bundle-collapser/issues/20 https://github.com/browserify/browserify/issues/1450 (cd client && browserify --no-dedupe -p bundle-collapser/plugin src/run-browser.js \ | ( [[ "$NODE_ENV" != "development" ]] && uglifyjs -cm || cat ) ) \ > $DEST/app.js +# Fonts are rewritten before CSS hashing; HTML is rendered only after app.js is hashed. +if [[ "$HASH_ASSETS" != "0" ]]; then + node scripts/hash-assets.js "$DEST" +fi +pug client/index.pug -o $DEST -O "$(node scripts/asset-manifest.js)" + # Pre-render notfound.html babel-node render-view.js '{"view":"error","error":"Page Not Found"}' > $DEST/notfound.html diff --git a/client/index.pug b/client/index.pug index a40194f7..e4aae793 100644 --- a/client/index.pug +++ b/client/index.pug @@ -4,7 +4,10 @@ doctype html - staticRoot = process.env.STATIC_ROOT || '' - pageTitle = prerender_title ? `${prerender_title} ยท ${baseTitle}` : baseTitle -- cssPath = t ? t`style.css` : 'style.css' +- assets = typeof assetManifest === 'object' && assetManifest || {} +- ltrPath = staticRoot + (assets['style.css'] || 'style.css') +- rtlPath = staticRoot + (assets['style-rtl.css'] || 'style-rtl.css') +- cssPath = t && t`style.css` === 'style-rtl.css' ? rtlPath : ltrPath - lang = t ? t.lang_id : 'en' - dir = t ? t`ltr` : 'ltr' @@ -29,7 +32,7 @@ head meta(name='viewport', content='width=device-width, initial-scale=1') link(rel='shortcut icon', type='image/png', href=staticRoot+'img/favicon.png') - link(rel='stylesheet', href=staticRoot+cssPath) + link(rel='stylesheet', href=cssPath, data-ltr=ltrPath, data-rtl=rtlPath) if process.env.CANONICAL_URL //- open search requires the absolute URL of the explorer @@ -53,5 +56,5 @@ body(lang=lang, dir=dir) .nojs-shim p This block explorer needs JavaScript to run. Please enable it and reload the page. - script(src=staticRoot+'app.js', async) + script(src=staticRoot+(assets['app.js'] || 'app.js'), async) != process.env.FOOT_HTML diff --git a/client/src/app.js b/client/src/app.js index a5ff5f08..1ea9f3e3 100644 --- a/client/src/app.js +++ b/client/src/app.js @@ -799,9 +799,14 @@ export default function main( // Click-to-copy if (navigator.clipboard) copy$.subscribe(text => navigator.clipboard.writeText(text)) - // Switch stylesheet based on current language - const stylesheet = document.querySelector('link[href="style.css"]') - t$.map(t => t`style.css`).distinctUntilChanged().subscribe(styleSrc => + // Keep the existing translation's RTL marker, but resolve URLs from the rendered link. + const stylesheet = document.querySelector('link[rel="stylesheet"][data-ltr]') + || document.querySelector('link[rel="stylesheet"]') // HTML cached before hashing was introduced + t$.map(t => { + const rtl = t`style.css` === 'style-rtl.css' + return stylesheet.getAttribute(rtl ? 'data-rtl' : 'data-ltr') + || (process.env.STATIC_ROOT || '') + (rtl ? 'style-rtl.css' : 'style.css') + }).distinctUntilChanged().subscribe(styleSrc => stylesheet.getAttribute('href') != styleSrc && (stylesheet.href = styleSrc)) // Apply language and text direction to root element diff --git a/client/src/driver/instascan.js b/client/src/driver/instascan.js index e946696e..63112690 100644 --- a/client/src/driver/instascan.js +++ b/client/src/driver/instascan.js @@ -1,6 +1,7 @@ import { Observable as O } from '../rxjs' const staticRoot = process.env.STATIC_ROOT || '' + , instascanAsset = process.env.INSTASCAN_ASSET || 'instascan.min.js' // check for WebRTC camera support if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) { @@ -11,11 +12,11 @@ if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) { if (loaded) return; loaded = true; const script = document.createElement('script') - script.src = `${staticRoot}instascan.min.js` + script.src = `${staticRoot}${instascanAsset}` document.body.appendChild(script) } - const Instascan$ = O.fromEvent(document.body, 'load', true).filter(e => e.target?.src?.endsWith('/instascan.min.js')).map(_ => window.Instascan).share() + const Instascan$ = O.fromEvent(document.body, 'load', true).filter(e => e.target?.src?.endsWith(`/${instascanAsset}`)).map(_ => window.Instascan).share() , Scanner$ = Instascan$.map(Instascan => Instascan.Scanner) , Camera$ = Instascan$.map(Instascan => Instascan.Camera) diff --git a/client/src/views/lander.js b/client/src/views/lander.js index da0d390f..19c7a1ba 100644 --- a/client/src/views/lander.js +++ b/client/src/views/lander.js @@ -16,7 +16,8 @@ const LandingPage = ({ t, ...S }) => { const logos = logoSources return layout(
- + {/* Read at render time: the prerender server sets it from the manifest at startup. */} +
diff --git a/contrib/runits/prerenderer.runit b/contrib/runits/prerenderer.runit index ceed22c1..841b7b77 100755 --- a/contrib/runits/prerenderer.runit +++ b/contrib/runits/prerenderer.runit @@ -6,6 +6,8 @@ export NODE_ENV=${NODE_ENV:-production} export API_URL=http://127.0.0.1/{NGINX_PATH}api export TZ=UTC export SOCKET_PATH=/var/prerender-http.sock +# Content-hashed asset names for this flavor's build (see README). +export ASSET_MANIFEST={STATIC_DIR}/asset-manifest.json exec 2>&1 exec /bin/bash -c 'source /root/.nvm/nvm.sh && cd /srv/explorer/source && npm run prerender-server -- --cluster {FLAVOR}' diff --git a/prerender-server/src/server.js b/prerender-server/src/server.js index ab979c33..1647adf7 100644 --- a/prerender-server/src/server.js +++ b/prerender-server/src/server.js @@ -7,6 +7,11 @@ import promClient from 'prom-client' import l10n from '../client/l10n' import render from '../client/run-server' +import { loadAssetManifest, exportAssetEnv } from '../../scripts/asset-manifest' + +// Load the flavor's manifest once; views read the lazy-script names from env at render time. +const assetManifest = loadAssetManifest() +exportAssetEnv(assetManifest) if (!process.env.API_URL) { throw new Error('API_URL environment variable is required but not defined.'); @@ -136,7 +141,8 @@ function doRender(req, res, next) { res.status(resp.status || 200) res.render(indexView, { - prerender_title: resp.title + assetManifest + , prerender_title: resp.title , prerender_html: resp.html , canon_url: canonBase ? canonBase + req.url : null , noscript: true diff --git a/render-view.js b/render-view.js index 1aa2f63b..37e6233a 100644 --- a/render-view.js +++ b/render-view.js @@ -9,7 +9,8 @@ state.t = l10n[state.lang || 'en'] state.page = { pathname: '', query: {} } require('pug').renderFile('client/index.pug', { - prerender_html: require('snabbdom-to-html')(view(state)) + assetManifest: require('./scripts/asset-manifest').loadAssetManifest() + , prerender_html: require('snabbdom-to-html')(view(state)) }, (err, html) => { if (err) throw err console.log(html) diff --git a/scripts/asset-manifest.js b/scripts/asset-manifest.js new file mode 100644 index 00000000..2b20eafe --- /dev/null +++ b/scripts/asset-manifest.js @@ -0,0 +1,35 @@ +const fs = require('fs') +const path = require('path') + +// Shared by the static build and both server-side renderers. Dev needs no manifest. +function loadAssetManifest() { + if (process.env.HASH_ASSETS === '0') return {} + const filename = process.env.ASSET_MANIFEST + || path.resolve(__dirname, '..', process.env.DEST || 'dist', 'asset-manifest.json') + try { + return JSON.parse(fs.readFileSync(filename, 'utf8')) + } catch (error) { + if (error.code === 'ENOENT') return {} + throw error + } +} + +// Lazy-loaded scripts are referenced from views through env vars. The browser bundle gets +// them from envify at build time; server-side renderers set them from the manifest. +const lazyAssetEnv = { + INSTASCAN_ASSET: 'instascan.min.js', + INFINITE_SCROLL_ASSET: 'js/infinite-scroll.js', +} + +function exportAssetEnv(manifest, env=process.env) { + for (const [ name, original ] of Object.entries(lazyAssetEnv)) { + if (!env[name] && manifest[original]) env[name] = manifest[original] + } + return env +} + +module.exports = { loadAssetManifest, exportAssetEnv } + +if (require.main === module) { + console.log(JSON.stringify({ assetManifest: loadAssetManifest() })) +} diff --git a/scripts/hash-assets.js b/scripts/hash-assets.js new file mode 100644 index 00000000..41630df0 --- /dev/null +++ b/scripts/hash-assets.js @@ -0,0 +1,65 @@ +const fs = require('fs') +const path = require('path') +const { createHash } = require('crypto') + +const lazyScripts = [ 'instascan.min.js', 'js/infinite-scroll.js' ] +const stylesheets = [ 'style.css', 'style-rtl.css' ] + +function hashAssets(destination, scriptsOnly=false) { + const root = path.resolve(destination) + , manifestPath = path.join(root, 'asset-manifest.json') + , previous = fs.existsSync(manifestPath) ? JSON.parse(fs.readFileSync(manifestPath, 'utf8')) : {} + , originals = Object.fromEntries(Object.entries(previous).map(([ original, hashed ]) => [ hashed, original ])) + , manifest = {} + + function copyHashed(relative) { + if (manifest[relative]) return manifest[relative] + const content = fs.readFileSync(path.join(root, relative)) + , hash = createHash('sha256').update(content).digest('hex').slice(0, 12) + , extension = path.extname(relative) + , hashed = `${relative.slice(0, -extension.length)}.${hash}${extension}` + fs.writeFileSync(path.join(root, hashed), content) + manifest[relative] = hashed + return hashed + } + + lazyScripts.forEach(copyHashed) + + if (!scriptsOnly) { + for (const stylesheet of stylesheets) { + const filename = path.join(root, stylesheet) + , css = fs.readFileSync(filename, 'utf8') + , rewritten = css.replace(/url\(\s*(['"]?)([^'"\)]+?)\1\s*\)/gi, (match, quote, url) => { + // Only local font URLs belong to phase 1. Preserve query strings/fragments. + if (/^(?:[a-z][a-z\d+.-]*:|\/\/)/i.test(url)) return match + const [ , pathname, suffix ] = url.match(/^([^?#]+)([?#].*)?$/) || [] + if (!pathname || !/\.(woff2?|ttf)$/i.test(pathname)) return match + const absolute = pathname.startsWith('/') + , relative = path.posix.normalize(absolute ? pathname.slice(1) : path.posix.join(path.posix.dirname(stylesheet), pathname)) + if (relative.startsWith('../')) throw new Error(`Font URL escapes output directory: ${url}`) + // Re-running the script must not hash an already hashed filename again. + const original = originals[relative] || relative + , hashed = copyHashed(original) + , rewrittenUrl = absolute ? '/' + hashed : path.posix.relative(path.posix.dirname(stylesheet), hashed) + return `url(${quote}${rewrittenUrl}${suffix || ''}${quote})` + }) + fs.writeFileSync(filename, rewritten) + copyHashed(stylesheet) + } + copyHashed('app.js') + } + + const sorted = Object.fromEntries(Object.entries(manifest).sort(([ a ], [ b ]) => a.localeCompare(b, 'en'))) + fs.writeFileSync(manifestPath, JSON.stringify(sorted, null, 2) + '\n') + return sorted +} + +module.exports = { hashAssets } + +if (require.main === module && process.env.HASH_ASSETS !== '0') { + const [ destination, mode ] = process.argv.slice(2) + if (!destination || (mode && mode !== '--scripts-only')) { + throw new Error('Usage: node scripts/hash-assets.js [--scripts-only]') + } + hashAssets(destination, mode === '--scripts-only') +} diff --git a/test/hash-assets.test.js b/test/hash-assets.test.js new file mode 100644 index 00000000..e249b598 --- /dev/null +++ b/test/hash-assets.test.js @@ -0,0 +1,153 @@ +const test = require('node:test') +const assert = require('node:assert/strict') +const fs = require('fs') +const path = require('path') +const { createHash } = require('crypto') +const { execFileSync } = require('child_process') +const pug = require('pug') +const { hashAssets } = require('../scripts/hash-assets') +const l10n = require('../client/src/l10n').default + +const root = path.resolve(__dirname, '..') +const digest = content => createHash('sha256').update(content).digest('hex').slice(0, 12) + +function fixture(t) { + fs.mkdirSync(path.join(root, 'dist'), { recursive: true }) + const directory = fs.mkdtempSync(path.join(root, 'dist', 'hash-test-')) + t.after(() => fs.rmSync(directory, { recursive: true, force: true })) + const files = { + 'app.js': 'app', + 'instascan.min.js': 'scanner', + 'js/infinite-scroll.js': 'scroll', + 'font/example.woff2': 'font two', + 'font/example.woff': 'font one', + 'font/example.ttf': 'font ttf', + 'font/unused.ttf': 'unused', + 'style.css': 'a{src:url("font/example.woff2?#v1")}b{src:url(./font/example.woff)}c{src:url(/font/example.ttf)}', + 'style-rtl.css': "a{src:url('font/example.ttf')}b{src:url(font/example.woff2)}", + 'img/example.svg': 'image', + 'libwally/wallycore.js': 'wally', + 'index.html': 'html', + 'robots.txt': 'robots', + 'opensearch.xml': 'search', + 'favicon.ico': 'icon', + } + for (const [ filename, content ] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(directory, filename)), { recursive: true }) + fs.writeFileSync(path.join(directory, filename), content) + } + return { directory, files } +} + +test('hashes only phase 1 assets, retains originals, and hashes final CSS bytes', t => { + const { directory, files } = fixture(t) + const prepared = hashAssets(directory, true) + assert.deepEqual(Object.keys(prepared), ['instascan.min.js', 'js/infinite-scroll.js']) + assert.equal(fs.readFileSync(path.join(directory, 'style.css'), 'utf8'), files['style.css']) + const manifest = hashAssets(directory) + assert.equal(Object.keys(manifest).length, 8) + for (const [ original, hashed ] of Object.entries(manifest)) { + const content = fs.readFileSync(path.join(directory, original)) + assert.deepEqual(content, fs.readFileSync(path.join(directory, hashed))) + assert.ok(hashed.endsWith(`.${digest(content)}${path.extname(original)}`)) + } + for (const filename of Object.keys(files).filter(filename => !manifest[filename])) { + assert.equal(fs.readFileSync(path.join(directory, filename), 'utf8'), files[filename]) + } + const css = fs.readFileSync(path.join(directory, manifest['style.css']), 'utf8') + assert.ok(css.includes(`url("${manifest['font/example.woff2']}?#v1")`)) + assert.ok(css.includes(`url(${manifest['font/example.woff']})`)) + assert.ok(css.includes(`url(/${manifest['font/example.ttf']})`)) + assert.deepEqual(hashAssets(directory), manifest, 'repeated hashing is idempotent') + + fs.appendFileSync(path.join(directory, 'font/example.woff2'), ' changed') + const changed = hashAssets(directory) + assert.notEqual(changed['font/example.woff2'], manifest['font/example.woff2']) + assert.notEqual(changed['style.css'], manifest['style.css']) + assert.notEqual(changed['style-rtl.css'], manifest['style-rtl.css']) + assert.equal(changed['app.js'], manifest['app.js']) +}) + +test('leaves external URLs unchanged and fails on missing local fonts', t => { + const { directory } = fixture(t) + const css = 'a{src:url(https://example.com/font.woff2)}b{src:url(//example.com/font.ttf)}c{src:url(data:font/woff;base64,AA)}' + fs.writeFileSync(path.join(directory, 'style.css'), css) + hashAssets(directory) + assert.equal(fs.readFileSync(path.join(directory, 'style.css'), 'utf8'), css) + fs.writeFileSync(path.join(directory, 'style.css'), 'a{src:url(font/missing.ttf)}') + assert.throws(() => hashAssets(directory), /ENOENT/) +}) + +test('HASH_ASSETS=0 skips hashing entirely and renderer loader falls back without a manifest', t => { + const { directory, files } = fixture(t) + execFileSync(process.execPath, ['scripts/hash-assets.js', directory], { + cwd: root, env: { ...process.env, HASH_ASSETS: '0' } + }) + assert.equal(fs.existsSync(path.join(directory, 'asset-manifest.json')), false) + for (const [ filename, content ] of Object.entries(files)) { + assert.equal(fs.readFileSync(path.join(directory, filename), 'utf8'), content) + } + const load = env => JSON.parse(execFileSync(process.execPath, ['scripts/asset-manifest.js'], { + cwd: root, env: { ...process.env, HASH_ASSETS: '1', ASSET_MANIFEST: path.join(directory, 'asset-manifest.json'), ...env } + })) + assert.deepEqual(load(), { assetManifest: {} }) + const manifest = hashAssets(directory) + assert.deepEqual(load(), { assetManifest: manifest }) + assert.deepEqual(load({ HASH_ASSETS: '0' }), { assetManifest: {} }) + assert.deepEqual(load({ ASSET_MANIFEST: '', DEST: directory }), { assetManifest: manifest }) + fs.writeFileSync(path.join(directory, 'asset-manifest.json'), '{broken') + assert.throws(() => load(), /SyntaxError/) +}) + +test('template resolves both directions and STATIC_ROOT consistently for every language', t => { + const previous = process.env.STATIC_ROOT + process.env.STATIC_ROOT = '/preview/assets/' + t.after(() => previous === undefined ? delete process.env.STATIC_ROOT : process.env.STATIC_ROOT = previous) + for (const assetManifest of [undefined, { + 'style.css': 'style.0123456789ab.css', + 'style-rtl.css': 'style-rtl.abcdef012345.css', + 'app.js': 'app.123456789abc.js' + }]) { + for (const [ lang, translate ] of Object.entries(l10n)) { + const html = pug.renderFile(path.join(root, 'client/index.pug'), { assetManifest, t: translate }) + , ltr = assetManifest ? assetManifest['style.css'] : 'style.css' + , rtl = assetManifest ? assetManifest['style-rtl.css'] : 'style-rtl.css' + , app = assetManifest ? assetManifest['app.js'] : 'app.js' + assert.ok(html.includes(`href="/preview/assets/${lang === 'he' ? rtl : ltr}"`), lang) + assert.ok(html.includes(`data-ltr="/preview/assets/${ltr}" data-rtl="/preview/assets/${rtl}"`)) + assert.ok(html.includes(`src="/preview/assets/${app}"`)) + } + } +}) + +test('exportAssetEnv sets lazy-script names from the manifest without overriding', () => { + const { exportAssetEnv } = require('../scripts/asset-manifest') + const manifest = { + 'instascan.min.js': 'instascan.min.0123456789ab.js', + 'js/infinite-scroll.js': 'js/infinite-scroll.ba9876543210.js', + } + assert.deepEqual(exportAssetEnv(manifest, {}), { + INSTASCAN_ASSET: 'instascan.min.0123456789ab.js', + INFINITE_SCROLL_ASSET: 'js/infinite-scroll.ba9876543210.js', + }) + assert.equal(exportAssetEnv(manifest, { INSTASCAN_ASSET: 'given.js' }).INSTASCAN_ASSET, 'given.js') + assert.deepEqual(exportAssetEnv({}, {}), {}) +}) + +test('server-rendered landing page reads the hashed infinite-scroll name at render time', t => { + const render = require('snabbdom-to-html') + const LandingPage = require('../client/src/views/lander').default + const previous = process.env.INFINITE_SCROLL_ASSET + t.after(() => { + if (previous === undefined) delete process.env.INFINITE_SCROLL_ASSET + else process.env.INFINITE_SCROLL_ASSET = previous + }) + const state = { t: l10n.en, page: { pathname: '/', query: {} } } + const scriptSrc = () => (render(LandingPage(state)).match(/