[WIP] chore(bundler): add tree shaking

tools/src/main/js/bundler/cli.js

@@ -102,7 +102,7 @@ const main = async (entryFile, includeDir) => {
     await writeFile(bundledFile, JSON.stringify(schema, null, 2));
 
     // 4. minify schema
-    subschemas(schema).forEach(dropComments);
+    subschemas(schema).forEach(([s]) => dropComments(s));
 
     await writeFile(minifiedFile, JSON.stringify({
         ...schema,
@@ -153,8 +153,8 @@ if (import.meta.main) {
             console.info(`bundler: embedded (${bundled.embedded.length}):`, bundled.embedded);
             console.info(`bundler: external (${bundled.external.length}):`, bundled.external);
 
-            console.info(`tree-shake: removed (${shaken.removed.length}):`, shaken.removed);
             console.info(`tree-shake: hollowed (${shaken.hollowed.length}):`, shaken.hollowed);
+            console.info(`tree-shake: removed (${shaken.removed.length}):`, shaken.removed);
 
             if (validationWarnings.length) {
                 console.warn(`validation warnings (${validationWarnings.length}):`, validationWarnings);

tools/src/main/js/bundler/src/bundle.js

@@ -1,7 +1,7 @@
 import {basename, dirname, relative, resolve} from 'node:path'
 
 import {JSON_SCHEMA_RE, getJsonfile, posixPath, findDuplicateValues} from './helpers/common.js';
-import {escapeJsonPointer, PREFIX as JP_PREFIX} from './helpers/json-pointer-spec.js';
+import {escapeJsonPointer, PREFIX as JP_PREFIX, ROOT_POINTER as JP_ROOT_POINTER} from './helpers/json-pointer-spec.js';
 import {
     ID_KEYWORD, NOT_KEYWORD, SCHEMA_KEYWORD, COMMENT_KEYWORD, DEFS_KEYWORD, REF_KEYWORDS,
     subschemas,
@@ -139,7 +139,7 @@ export const bundle = async (entryFile, includeFiles, targetFile) => {
                 rewiredRS = {
                     path: null,
                     frag: (refFile === entryFile && !frag?.startsWith(FRAG_DEFS_PREFIX)
-                            ? ''
+                            ? JP_ROOT_POINTER
                             : `${FRAG_DEFS_PREFIX}${escapeJsonPointer(moduleName)}`
                     ) + (frag ?? '')
                 };
@@ -156,7 +156,7 @@ export const bundle = async (entryFile, includeFiles, targetFile) => {
                 externals.add(rewired);
             }
         }
-        subschemas(schema).forEach(s => rewireRefs(s, sourceFile));
+        subschemas(schema).forEach(([s]) => rewireRefs(s, sourceFile));
     }
 
     const schema = await getJsonfile(entryFile);
@@ -178,6 +178,8 @@ export const bundle = async (entryFile, includeFiles, targetFile) => {
             [NOT_KEYWORD]: {[COMMENT_KEYWORD]: HOLLOW_COMMENT}
         };
         embeddedMap.set(`${entryFile}${REF_FRAG_SEP}${JP_PREFIX}${DEFS_KEYWORD}`, `${REF_FRAG_SEP}${FRAG_DEFS_PREFIX}${escapeJsonPointer(entryModuleName)}`);
+    } else {
+        delete schema[DEFS_KEYWORD][entryModuleName];
     }
 
     for (const [includeFile, moduleName] of fileModuleNames) {

tools/src/main/js/bundler/src/cleanup.js

@@ -1,4 +1,14 @@
-import {COMMENT_KEYWORD, subschemas} from './helpers/json-schema-spec.js';
+import {objectGetOwnPath} from "./helpers/common.js";
+import {isJsonPointer, jsonPointer2stack, jsonPointer4stack, PREFIX as JP_PREFIX} from "./helpers/json-pointer-spec.js";
+import {
+    ANCHOR_KEYWORDS,
+    COMMENT_KEYWORD,
+    DEFS_KEYWORD,
+    isSchemaObject, NOT_KEYWORD,
+    REF_FRAG_SEP,
+    REF_KEYWORDS,
+    subschemas
+} from './helpers/json-schema-spec.js';
 
 /**
  * Remove $comment in place
@@ -7,18 +17,22 @@ import {COMMENT_KEYWORD, subschemas} from './helpers/json-schema-spec.js';
  */
 export const dropComments = (schema) => {
     delete schema[COMMENT_KEYWORD];
-    subschemas(schema).forEach(dropComments);
+    subschemas(schema).forEach(([s]) => dropComments(s));
 }
 
 const HOLLOW_COMMENT =
     'This schema is hollow: it was not referenced ' +
     'and only remains as a container for definitions that are still in use. ' +
     'Itself must not be used for validation.';
 
+/** @typedef {import('./helpers/json-pointer-spec.js').JsonPointerStack} JsonPointerStack */
+
+
 /**
  * @typedef {object} TreeShakeResult
- * @property {string[]} hollowed
- * @property {string[]} removed
+ * @property {JsonPointer[]} reached
+ * @property {JsonPointer[]} hollowed
+ * @property {JsonPointer[]} removed
  */
 
 /**
@@ -34,13 +48,112 @@ const HOLLOW_COMMENT =
  * @returns {TreeShakeResult}
  */
 export const treeShake = (schema) => {
-    /** @type {string[]} */
+    /** @type {JsonPointer[]} */
     const removed = [];
-    /** @type {string[]} */
+    /** @type {JsonPointer[]} */
     const hollowed = [];
 
-    // TODO: tree-shake schema in place
-    // will be added in a dedicated PR later.
+    /**
+     * JSON pointers of all reached schemas.
+     * @type {Set<JsonPointer>}
+     */
+    const reached = new Set();
+
+    /**
+     * Mark phase: walk a schema and everything it references.
+     *
+     * Does NOT descend into `$defs` - definitions are only reached via refs.
+     *
+     * @param {Readonly<*>} s
+     * @param {JsonPointerStack} stack
+     * @returns {void}
+     */
+    const markReached = (s, stack) => {
+        const p = jsonPointer4stack(stack);
+        if (reached.has(p)) return;
+        reached.add(p);
+
+        for (const k of REF_KEYWORDS) {
+            const v = s[k];
+            if (typeof v !== 'string' || !v.startsWith(REF_FRAG_SEP)) continue;
+            const frag = v.slice(REF_FRAG_SEP.length);
+            if (!isJsonPointer(frag)) {
+                // anchors are not supported - leave untouched
+                continue;
+            }
+            const tStack = jsonPointer2stack(frag);
+            const tSchema = objectGetOwnPath(schema, tStack);
+            if (isSchemaObject(tSchema)) {
+                markReached(tSchema, tStack);
+            }
+        }
+
+        subschemas(s).forEach(([sub, rel]) => {
+            if (rel[0] === DEFS_KEYWORD) return;
+            markReached(sub, [...stack, ...rel]);
+        });
+    };
+
+    /**
+     * Sweep phase: remove or hollow non-reached definitions.
+     * @param {*} s
+     * @param {JsonPointerStack} stack
+     * @returns {void}
+     */
+    const sweep = (s, stack) => {
+        const defs = s[DEFS_KEYWORD];
+        if (!defs) return;
+        for (const [dName, def] of Object.entries(defs)) {
+            if (!isSchemaObject(def)) continue;
+
+            const dStack = [...stack, DEFS_KEYWORD, dName];
+            const dPtr = jsonPointer4stack(dStack);
+            if (reached.has(dPtr)) {
+                sweep(def, dStack);
+                continue;
+            }
+            if (ANCHOR_KEYWORDS.values().some(k => def.hasOwnProperty(k))) {
+                // we do NOT support anchors, yet -> leave untouched for now
+                continue;
+            }
+            // definition itself is unused
+
+            const belowPrefix = `${dPtr}${JP_PREFIX}`;
+            const below = Object.freeze(reached.values().filter(r => r.startsWith(belowPrefix)).toArray());
+            if (below.length === 0) {
+                delete defs[dName];
+                removed.push(`${REF_FRAG_SEP}${dPtr}`);
+                continue;
+            }
+            // something beneath is still in use
+
+            sweep(def, dStack);
+
+            const belowPrefixDefs = `${belowPrefix}${DEFS_KEYWORD}${JP_PREFIX}`;
+            if (below.every(r => r.startsWith(belowPrefixDefs))) {
+                // only $defs sub-paths are referenced - make hollow
+                Object.keys(def).forEach(k => {
+                    if (k === DEFS_KEYWORD) return;
+                    delete def[k];
+                });
+                def[NOT_KEYWORD] = {[COMMENT_KEYWORD]: HOLLOW_COMMENT};
+                hollowed.push(`${REF_FRAG_SEP}${dPtr}`);
+            }
+        }
+
+        if (Object.keys(defs).length === 0) {
+            delete s[DEFS_KEYWORD];
+        }
+    };
+
+    markReached(schema, []);
+    Object.freeze(reached);
+
+    sweep(schema, []);
 
-    return {hollowed, removed};
+    return {
+        reached: Array.from(reached, r => `${REF_FRAG_SEP}${r}`).sort(),
+        hollowed: hollowed.sort(),
+        removed: removed.sort()
+    };
 };

tools/src/main/js/bundler/src/helpers/common.js

@@ -30,23 +30,31 @@ export const getJsonfile = async (file) =>
 export const posixPath = (p) =>
     p.replaceAll(sep, '/');
 
+/**
+ * Sentinel returned by {@link objectGetOwnPath} when a path does not exist.
+ *
+ * A Symbol is used, since `undefined`/`null` may be legit values.
+ */
+export const PATH_NOT_FOUND = Symbol('PATH_NOT_FOUND');
 
 /**
+ * Get the value at a path of own properties.
+ *
  * @param {Readonly<*>} o
  * @param {Iterable<string>} pathStack
- * @returns {boolean}
+ * @returns {*|typeof PATH_NOT_FOUND} the value, or {@link PATH_NOT_FOUND} if the path does not exist
  */
-export const objectHasOwnPath = (o, pathStack) => {
+export const objectGetOwnPath = (o, pathStack) => {
     let c = o;
     for (const s of pathStack) {
         try {
-            if (!Object.hasOwn(c, s)) return false;
+            if (!Object.hasOwn(c, s)) return PATH_NOT_FOUND;
         } catch { // TypeErrors and such.
-            return false;
+            return PATH_NOT_FOUND;
         }
         c = c[s];
     }
-    return true;
+    return c;
 };
 
 

tools/src/main/js/bundler/src/helpers/json-pointer-spec.js

@@ -7,11 +7,17 @@
  * @typedef {string} JsonPointer
  */
 
+/**
+ * Unescaped JSON Pointer token.
+ * @typedef {string} JsonPointerToken
+ */
+
 /**
  * Stack of unescaped JSON Pointer tokens.
- * @typedef {string[]} JsonPointerStack
+ * @typedef {JsonPointerToken[]} JsonPointerStack
  */
 
+export const ROOT_POINTER = '';
 
 export const PREFIX = '/';
 const PREFIX_ESC = '~1';
@@ -23,7 +29,7 @@ const TILDE_ESC = '~0';
  * @returns {boolean}
  */
 export const isJsonPointer = (s) =>
-    s.startsWith(PREFIX);
+    s === ROOT_POINTER || s.startsWith(PREFIX);
 
 /**
  * JSON Pointer to stack.
@@ -39,7 +45,7 @@ export const jsonPointer2stack = (p) =>
  * @returns {JsonPointer}
  */
 export const jsonPointer4stack = (s) =>
-    PREFIX + s.map(escapeJsonPointer).join(PREFIX);
+    s.map(t => `${PREFIX}${escapeJsonPointer(t)}`).join('')
 
 
 /**

tools/src/main/js/bundler/src/helpers/json-schema-spec.js

@@ -47,16 +47,26 @@ export const isSchemaObject = (o) =>
 /**
  * Yield immediate subschemas, never property maps or literal data.
  * @param {Readonly<*>} s schema
- * @yields {*}
+ * @yields {[*, import('./json-pointer-spec.js').JsonPointerStack]} schema and pointer stack
  */
 export const subschemas = function* (s) {
     for (const [k, v] of Object.entries(s)) {
         if (SINGLE_SCHEMA.has(k)) {
-            if (isSchemaObject(v)) yield v;
-        } else if (SCHEMA_ARRAY.has(k)) {
-            if (Array.isArray(v)) yield* v.filter(isSchemaObject);
-        } else if (SCHEMA_MAP.has(k)) {
-            if (isSchemaObject(v)) yield* Object.values(v).filter(isSchemaObject);
+            if (isSchemaObject(v)) yield [v, [k]];
+            continue;
+        }
+        if (SCHEMA_ARRAY.has(k)) {
+            if (!Array.isArray(v)) continue;
+            for (const [ak, av] of v.entries()) {
+                if (isSchemaObject(av)) yield [av, [k, String(ak)]];
+            }
+            continue;
+        }
+        if (SCHEMA_MAP.has(k)) {
+            if (!isSchemaObject(v)) continue;
+            for (const [mk, mv] of Object.entries(v)) {
+                if (isSchemaObject(mv)) yield [mv, [k, mk]];
+            }
         }
     }
 };

tools/src/main/js/bundler/src/sanity.js

@@ -1,6 +1,6 @@
-import {objectHasOwnPath} from "./helpers/common.js";
+import {objectGetOwnPath, PATH_NOT_FOUND} from "./helpers/common.js";
 import {REF_FRAG_SEP, REF_KEYWORDS, subschemas} from './helpers/json-schema-spec.js';
-import {isJsonPointer, jsonPointer2stack} from './helpers/json-pointer-spec.js'
+import {isJsonPointer, jsonPointer2stack, ROOT_POINTER as JP_ROOT_POINTER} from './helpers/json-pointer-spec.js'
 
 
 /**
@@ -45,15 +45,14 @@ const validateRefs = (schema) => {
                 warnings.add(`Unchecked external ref: ${v}`);
             }
         }
-        subschemas(schema).forEach(gatherRefs);
+        subschemas(schema).forEach(([s]) => gatherRefs(s));
     }
 
     gatherRefs(schema);
 
     for (const ref of localRefs) {
-        if (ref === '') continue;
         if (isJsonPointer(ref)) {
-            if (!objectHasOwnPath(schema, jsonPointer2stack(ref))) {
+            if (objectGetOwnPath(schema, jsonPointer2stack(ref)) === PATH_NOT_FOUND) {
                 errors.add(`Missing local ref: ${REF_FRAG_SEP}${ref}`);
             }
         } else {

tools/src/main/js/bundler/test/fixtures/modules/used-whole.schema.json

@@ -52,9 +52,9 @@
       "description": "An integer value.",
       "type": "integer"
     },
-    "unusedButKept": {
-      "$comment": "Not referenced anywhere, but kept because the module is used as a whole.",
-      "title": "Unused but kept",
+    "unused": {
+      "$comment": "Not referenced anywhere, dropped even though the module is used as a whole.",
+      "title": "Unused",
       "description": "A flag.",
       "type": "boolean"
     },

tools/src/main/js/bundler/test/snapshot/main-bundled.min.schema.json

@@ -1 +1 @@
-{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"http://example.com/test/main-bundled.min.schema.json","$comment":"FIXTURE: root schema; entry point of the bundle. This $comment is the only one that survives minification. The root title/description must survive in both outputs.","title":"Main","description":"Root document of the bundler test fixtures.","type":"object","properties":{"$comment":{"type":"string","title":"Comment"},"whole":{"title":"Whole","description":"A module referenced in its entirety.","$ref":"#/$defs/used-whole"},"sub":{"title":"Sub","description":"A single definition of another module.","$ref":"#/$defs/used-subdefs-only/$defs/kept"},"local":{"title":"Local","description":"A definition of this very schema.","$ref":"#/$defs/main/$defs/mainLocal"}},"$defs":{"dynamic-anchor":{"$dynamicAnchor":"node","title":"Dynamic anchor","description":"A recursive tree node, addressable via a dynamic anchor.","type":"object","properties":{"children":{"title":"Children","description":"The child nodes of this node.","type":"array","items":{"title":"Child","description":"A nested node.","$ref":"#/$defs/dynamic-anchor"}}}},"main":{"$defs":{"mainLocal":{"title":"Main local","description":"A local string definition of the root schema.","type":"string"},"mainUnused":{"title":"Main unused","description":"A definition nothing refers to.","type":"null"}},"not":{}},"unused":{"title":"Unused","description":"A module that nothing reachable refers to.","type":"object","properties":{"pullsNothingIn":{"title":"Pulls nothing in","description":"A flag defined by another module.","$ref":"#/$defs/used-whole/$defs/unusedButKept"}}},"used-subdefs-only":{"title":"Used sub-defs only","description":"A module of which only individual definitions are referenced.","type":"object","properties":{"body":{"title":"Body","description":"The body of this module.","$ref":"#/$defs/unused"}},"$defs":{"kept":{"title":"Kept","description":"A definition that is reached directly from the root schema.","type":"object","properties":{"shared":{"title":"Shared","description":"A value defined by another module.","$ref":"#/$defs/used-whole/$defs/sharedByOther"},"fromMain":{"title":"From main","description":"A value defined by the root schema.","$ref":"#/$defs/main/$defs/mainLocal"},"chain":{"title":"Chain","description":"A value defined further down in this module.","$ref":"#/$defs/used-subdefs-only/$defs/keptTransitively"}}},"keptTransitively":{"title":"Kept transitively","description":"A string value.","type":"string"},"dropped":{"title":"Dropped","description":"A definition nothing reachable refers to.","type":"null"}}},"used-whole":{"title":"Used whole","description":"A module that is referenced in its entirety.","type":"object","properties":{"name":{"title":"Name","description":"A display name.","type":"string"},"inner":{"title":"Inner","description":"Sibling annotations next to a $ref: title/description are kept, while the sibling $comment is removed in the minified output.","$ref":"#/$defs/used-whole/$defs/inner"},"license":{"title":"License","description":"The license of this item.","$ref":"externals/not-bundled.schema.json#/$defs/licenseId"},"remote":{"title":"Remote","description":"A thing defined by a remote schema.","$ref":"https://example.com/remote.schema.json#/$defs/thing"},"inArray":{"title":"In array","description":"A composition of several referenced schemas.","allOf":[{"$ref":"#/$defs/used-whole/$defs/inner"},{"$ref":"#/$defs/used-subdefs-only/$defs/keptTransitively"},{"$ref":"externals/not-bundled.schema.json#/$defs/licenseId"}]}},"$defs":{"inner":{"title":"Inner","description":"An integer value.","type":"integer"},"unusedButKept":{"title":"Unused but kept","description":"A flag.","type":"boolean"},"sharedByOther":{"title":"Shared by other","description":"A numeric value shared across modules.","type":"number"}}}}}
+{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"http://example.com/test/main-bundled.min.schema.json","$comment":"FIXTURE: root schema; entry point of the bundle. This $comment is the only one that survives minification. The root title/description must survive in both outputs.","title":"Main","description":"Root document of the bundler test fixtures.","type":"object","properties":{"$comment":{"type":"string","title":"Comment"},"whole":{"title":"Whole","description":"A module referenced in its entirety.","$ref":"#/$defs/used-whole"},"sub":{"title":"Sub","description":"A single definition of another module.","$ref":"#/$defs/used-subdefs-only/$defs/kept"},"local":{"title":"Local","description":"A definition of this very schema.","$ref":"#/$defs/main/$defs/mainLocal"}},"$defs":{"dynamic-anchor":{"$dynamicAnchor":"node","title":"Dynamic anchor","description":"A recursive tree node, addressable via a dynamic anchor.","type":"object","properties":{"children":{"title":"Children","description":"The child nodes of this node.","type":"array","items":{"title":"Child","description":"A nested node.","$ref":"#/$defs/dynamic-anchor"}}}},"main":{"$defs":{"mainLocal":{"title":"Main local","description":"A local string definition of the root schema.","type":"string"}},"not":{}},"used-subdefs-only":{"$defs":{"kept":{"title":"Kept","description":"A definition that is reached directly from the root schema.","type":"object","properties":{"shared":{"title":"Shared","description":"A value defined by another module.","$ref":"#/$defs/used-whole/$defs/sharedByOther"},"fromMain":{"title":"From main","description":"A value defined by the root schema.","$ref":"#/$defs/main/$defs/mainLocal"},"chain":{"title":"Chain","description":"A value defined further down in this module.","$ref":"#/$defs/used-subdefs-only/$defs/keptTransitively"}}},"keptTransitively":{"title":"Kept transitively","description":"A string value.","type":"string"}},"not":{}},"used-whole":{"title":"Used whole","description":"A module that is referenced in its entirety.","type":"object","properties":{"name":{"title":"Name","description":"A display name.","type":"string"},"inner":{"title":"Inner","description":"Sibling annotations next to a $ref: title/description are kept, while the sibling $comment is removed in the minified output.","$ref":"#/$defs/used-whole/$defs/inner"},"license":{"title":"License","description":"The license of this item.","$ref":"externals/not-bundled.schema.json#/$defs/licenseId"},"remote":{"title":"Remote","description":"A thing defined by a remote schema.","$ref":"https://example.com/remote.schema.json#/$defs/thing"},"inArray":{"title":"In array","description":"A composition of several referenced schemas.","allOf":[{"$ref":"#/$defs/used-whole/$defs/inner"},{"$ref":"#/$defs/used-subdefs-only/$defs/keptTransitively"},{"$ref":"externals/not-bundled.schema.json#/$defs/licenseId"}]}},"$defs":{"inner":{"title":"Inner","description":"An integer value.","type":"integer"},"sharedByOther":{"title":"Shared by other","description":"A numeric value shared across modules.","type":"number"}}}}}