diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4a336b79..c522429a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -193,6 +193,12 @@ The `Node.js` engine requirements are updated on a predictable biannual schedule ### Project maintenance +#### Collection reseeding +When PatternFly releases new documentation or component features, reseed local catalog data: +- `npm run build:collections`: Runs the complete crawler and manifest update pipeline and runs collection tests. +- `npm run build:collections-api`: Updates `src/collection.patternFlyApi.json` and outputs CSV diffs to `reports/`. +- `npm run build:collections-docs`: Updates `src/docs.json` with latest repository hashes and prunes API duplicates. + For information on project development maintenance tasks, refer to [Project development maintenance in docs/development.md](./docs/development.md#project-development-maintenance). ## AI agent diff --git a/cspell.config.json b/cspell.config.json index f04e8ace..13842167 100644 --- a/cspell.config.json +++ b/cspell.config.json @@ -27,6 +27,7 @@ "sandboxing", "sparkline", "streamable", + "symref", "tmpfs", "treeify", "unrepresentable", diff --git a/docs/development.md b/docs/development.md index 2917730d..31b72efe 100644 --- a/docs/development.md +++ b/docs/development.md @@ -460,19 +460,32 @@ The server maintains multiple record collections that provide documentation and The original curated Markdown collection. It pins remote repository URLs to specific Git commit SHAs to guarantee consistent documentation and remove upstream breaking changes. -- **Update workflow**: Follow the [add-docs-links skill](../guidelines/skills/add-docs-links/SKILL.md) to add, update, or remove entries. +- **Update workflow**: Follow the [add-docs-links skill](../guidelines/skills/add-docs-links/SKILL.md) for manual additions/edits, or run the automated reseeding script: + ```bash + npm run build:collections-docs + ``` +- **Automated synchronization**: The Docs reseeding pipeline (`tooling/collections/update.patternFlyDocs.ts`) automatically resolves the latest commit SHAs across referenced GitHub repositories, prunes redundant AI guideline entries superseded by high-quality API endpoints, recalculates `meta.totalEntries` and `meta.totalDocs`, and generates an audit diff report at `reports/YYYYMMDD-collection.patternFlyDocs.report.csv`. - **Validation**: Ensure all raw URLs are HTTPS, match the domain whitelist in `src/options.defaults.ts`, and return 2xx responses. -- **Testing**: Run `npm test` and update `baseHashes` and the Repository Breakdown table in `src/__tests__/docs.json.test.ts` whenever refs change. +- **Testing**: Run `npm run test:collections` or `npm test`, updating `baseHashes` and the Repository Breakdown table in `src/__tests__/docs.json.test.ts` when refs change. #### 2. Pre-built API collection (`src/collection.patternFlyApi.json`) The server packages pre-seeded API endpoints to enable instant server startup without network overhead. -- **Update command**: +- **Update commands**: ```bash + # Reseed both API and Docs collections and run collection test suites npm run build:collections + + # Reseed only the API collection + npm run build:collections-api + + # Run collection test suites only + npm run test:collections ``` -- **Execution**: Crawls live PatternFly API endpoints, filters quality records, updates `src/collection.patternFlyApi.json`, and executes collection-specific tests. +- **Execution**: Crawls live PatternFly API endpoints via `apiSpider`, filters low-quality records, compresses content into `src/collection.patternFlyApi.json`, and outputs structured console diffs and timestamped CSV audit reports to `reports/YYYYMMDD-collection.patternFlyApi.report.csv`. +- **Configuration & Environment Overrides**: + - `UPDATE_COLLECTIONS=true`: Required environment flag to execute collection builds directly. - **When to run**: When PatternFly publishes new releases, when updating metadata/quality filters, or as part of a general maintenance cycle. #### 3. Component schemas (`@patternfly/patternfly-component-schemas`) diff --git a/eslint.config.js b/eslint.config.js index ad204b0b..c13bafca 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -127,7 +127,9 @@ export default [ { files: [ 'scripts/**/*.ts', - 'scripts/**/*.js' + 'scripts/**/*.js', + 'tooling/**/*.ts', + 'tooling/**/*.js' ], rules: { 'no-console': 0, diff --git a/guidelines/README.md b/guidelines/README.md index 860c2cc0..864ac18a 100644 --- a/guidelines/README.md +++ b/guidelines/README.md @@ -37,6 +37,7 @@ Agents should use these phrases as signals to consult specific documentation and | **"review development guide"** | Review `docs/development.md` for CLI, API, and plugin authoring. | | **"create an example tool plugin"** | Review `guidelines/agent_coding.md`, `docs/development.md`, `docs/examples/*`, and `src/*` for context, coding standards, and existing example formats. | | **"add documentation links"** / **"add doc entries"** / **"register docs"** / **"update docs.json"** / **"contribute to docs.json"** | Follow `guidelines/skills/add-docs-links/SKILL.md`: docs.json format, duplicate check, raw URL confirmation, then run unit tests and update meta. | +| **"reseed collections"** / **"update collections"** / **"build collections"** / **"refresh docs manifest"** | Follow `docs/development.md#how-collection-and-source-data-get-updated`: run `npm run build:collections` (or `build:collections-api` / `build:collections-docs`) to sync API records and docs manifests. | | **"review zod"** / **"zod upgrade"** / **"zod integration review"** / **"zod update report"** | Follow `guidelines/skills/review-zod-integration/SKILL.md`: release notes vs codebase, tests, report at `reports/YYYYMMDD-HHMMSS-zod-{semver}-update-report.md` (UTC). | | **"troubleshoot server"** / **"debug server"** | Review `docs/usage.md#troubleshooting` and the PatternFly MCP server resource `patternfly://context` | diff --git a/jest.config.ts b/jest.config.ts index cb22edca..dc72920a 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -72,9 +72,10 @@ export default { ...baseConfig }, { - displayName: 'collections', - roots: ['/tests/scripts'], - testMatch: ['/tests/scripts/**/*collection*.test.ts'], + displayName: 'tooling:collections', + roots: ['/tooling/collections'], + testMatch: ['/tooling/collections/**/__tests__/**/*.test.ts'], + setupFilesAfterEnv: ['/jest.setupTests.ts'], ...baseConfig }, { diff --git a/package.json b/package.json index 494a128f..8d62c468 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,9 @@ "build": "npm run build:clean; npm run test:types; pkgroll", "build:clean": "rm -rf dist", "build:watch": "npm run build -- --watch", - "build:collections": "UPDATE_COLLECTIONS=true tsx ./scripts/update.collection.patternFlyApi.ts && NODE_OPTIONS='--experimental-vm-modules' jest --selectProjects collections", + "build:collections": "npm run build:collections-api && npm run build:collections-docs && jest --selectProjects tooling:collections", + "build:collections-api": "UPDATE_COLLECTIONS=true tsx ./tooling/collections/update.patternFlyApi.ts", + "build:collections-docs": "UPDATE_COLLECTIONS=true tsx ./tooling/collections/update.patternFlyDocs.ts", "container:build": "bash ./scripts/container.build.sh", "container:start": "bash ./scripts/container.run.sh", "release": "changelog --non-cc --link-url https://github.com/patternfly/patternfly-mcp.git", @@ -45,7 +47,8 @@ "test:audit": "NODE_OPTIONS='--experimental-vm-modules' jest --selectProjects audit", "test:audit-container": "npm run container:build && jest --selectProjects audit:container", "test:ci": "npm test -- --coverage", - "test:collections": "NODE_OPTIONS='--experimental-vm-modules' jest --selectProjects collections", + "test:collections": "jest --selectProjects tooling:collections", + "test:collections-dev": "npm run test:collections -- --watchAll", "test:dev": "npm test -- --watchAll", "test:integration": "npm run build && jest --selectProjects package && NODE_OPTIONS='--experimental-vm-modules' jest --selectProjects e2e", "test:integration-dev": "npm run test:integration -- --watchAll", @@ -53,7 +56,7 @@ "test:lint-fix": "eslint . --fix", "test:package": "npm run build && jest --selectProjects package", "test:spell-docs": "cspell './README.md' './CONTRIBUTING.md' './GOVERNANCE.md' './SECURITY.md' './docs/**/*.md' './guidelines/**/*.md' './docs/**/*.ts' './docs/**/*.js' --config ./cspell.config.json --fail-fast", - "test:spell": "cspell './src/**/*.ts' './tests/**/*.ts' --exclude './src/**/*test*' --exclude './tests/**/*test*' --config ./cspell.config.json --fail-fast", + "test:spell": "cspell './src/**/*.ts' './tests/**/*.ts' './tooling/**/*.ts' --exclude './src/**/*test*' --exclude './tests/**/*test*' --exclude './tooling/**/*test*' --config ./cspell.config.json --fail-fast", "test:types": "tsc --noEmit", "prepare": "npm run build", "prepack": "npm run build" diff --git a/src/__tests__/docs.json.test.ts b/src/__tests__/docs.json.test.ts index 20db5cf3..d4054f13 100644 --- a/src/__tests__/docs.json.test.ts +++ b/src/__tests__/docs.json.test.ts @@ -74,7 +74,7 @@ describe('docs.json', () => { * * | Repository | Hash / Branch | Count | Type | * |---|---|---|---| - * | uxd-ai-helpers | e8cca17430a8ccb062ed1878073165417a081b34 | 6 | **Primary** | + * | uxd-ai-helpers | e8cca17430a8ccb062ed1878073165417a081b34 | 6 | One-off | * | patternfly-cli | ce032cd16ddb90c540cb4f18c6830e190cd9e3e9 | 1 | One-off | * | patternfly-elements | 402b3b0e7ed73cb2aa21531e0eab4216c2211212 | 1 | One-off | * | patternfly-mcp | 4e1cd7c636f6ac17c851f54a920549e5377f3809 | 5 | One-off | diff --git a/tests/audit/docs.audit.test.ts b/tests/audit/docs.audit.test.ts index e410dd68..ec400ba5 100644 --- a/tests/audit/docs.audit.test.ts +++ b/tests/audit/docs.audit.test.ts @@ -1,11 +1,22 @@ import { randomInt } from 'node:crypto'; +import { existsSync, readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; import { jest } from '@jest/globals'; -import docs from '../../src/docs.json'; +import { type PatternFlyMcpDocsCatalog } from '../../src/docs.embedded'; import { checkUrl } from './utils/checkUrl'; describe('Documentation Link Audit', () => { let auditSet: string[] = []; + const catalogPath = resolve( + process.cwd(), + process.env.DOCS_COLLECTION_PATH || 'src/docs.json' + ); + + const docs: PatternFlyMcpDocsCatalog = existsSync(catalogPath) + ? JSON.parse(readFileSync(catalogPath, 'utf-8')) + : { meta: { totalEntries: 0, totalDocs: 0, source: '' }, docs: {} }; + /** * Extracts and returns a prefix based on the provided URL. * - If the hostname is not "raw.githubusercontent.com", the hostname is returned. diff --git a/tooling/collections/__tests__/csv.test.ts b/tooling/collections/__tests__/csv.test.ts new file mode 100644 index 00000000..f1a4d54d --- /dev/null +++ b/tooling/collections/__tests__/csv.test.ts @@ -0,0 +1,174 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import { + escapeCsvField, + formatCsv, + generateDiffCsv, + getDefaultReportPath, + getReportDatePrefix, + saveCsvReport +} from '../csv'; + +jest.mock('node:fs/promises', () => ({ + ...jest.requireActual('node:fs/promises'), + mkdir: jest.fn(), + writeFile: jest.fn() +})); + +const mockMkdir = mkdir as jest.MockedFunction; +const mockWriteFile = writeFile as jest.MockedFunction; + +describe('getReportDatePrefix', () => { + it('should format a given date into YYYYMMDD- prefix format', () => { + const fixedDate = new Date(2026, 9, 9); // Oct 9, 2026 + + expect(getReportDatePrefix(fixedDate)).toBe('20261009-'); + }); + + it('should properly zero-pad single digit months and days', () => { + const singleDigitDate = new Date(2026, 0, 5); // Jan 5, 2026 + + expect(getReportDatePrefix(singleDigitDate)).toBe('20260105-'); + }); + + it('should generate a valid 9-character prefix matching /^[0-9]{8}-$/ when using current date', () => { + expect(getReportDatePrefix()).toMatch(/^[0-9]{8}-$/); + }); +}); + +describe('getDefaultReportPath', () => { + it('should resolve default path containing the timestamp prefix and filename', () => { + const fixedDate = new Date(2026, 9, 9); + const apiFileName = 'loremIpsum.report.csv'; + const docFileName = 'dolorSit.report.csv'; + const apiPath = getDefaultReportPath(apiFileName, fixedDate); + const docsPath = getDefaultReportPath(docFileName, fixedDate); + + expect(apiPath).toMatch(new RegExp(`reports/20261009-${apiFileName}$`)); + expect(docsPath).toMatch(new RegExp(`reports/20261009-${docFileName}$`)); + }); +}); + +describe('escapeCsvField', () => { + it.each([ + { description: 'plain string', input: 'normal', expected: 'normal' }, + { description: 'integer number', input: 123, expected: '123' }, + { description: 'zero', input: 0, expected: '0' }, + { description: 'boolean true', input: true, expected: 'true' }, + { description: 'boolean false', input: false, expected: 'false' }, + { description: 'null', input: null, expected: '' }, + { description: 'undefined', input: undefined, expected: '' }, + { description: 'comma delimiter', input: 'with,comma', expected: '"with,comma"' }, + { description: 'double quotes', input: 'with "quotes"', expected: '"with ""quotes"""' }, + { description: 'newline character', input: 'with\nnewline', expected: '"with\nnewline"' }, + { description: 'carriage return character', input: 'with\rreturn', expected: '"with\rreturn"' }, + { description: 'CRLF characters', input: 'with\r\nboth', expected: '"with\r\nboth"' }, + { description: 'combined quotes, commas, and newlines', input: 'all "in, one"\nline', expected: '"all ""in, one""\nline"' }, + { description: 'formula injection (=)', input: '=SUM(1+1)', expected: "'=SUM(1+1)" }, + { description: 'formula injection (+)', input: '+123', expected: "'+123" }, + { description: 'formula injection (-)', input: '-456', expected: "'-456" }, + { description: 'formula injection (@)', input: '@lookup', expected: "'@lookup" }, + { description: 'formula injection (tab)', input: '\ttabPrefix', expected: "'\ttabPrefix" }, + { description: 'formula injection (carriage return prefix)', input: '\rreturnPrefix', expected: '"\'\rreturnPrefix"' } + ])('should correctly escape field value, $description', ({ input, expected }) => { + expect(escapeCsvField(input)).toBe(expected); + }); + + it.each([ + { description: 'equals formula', input: '=SUM(1+1)', expected: '=SUM(1+1)' }, + { description: 'plus prefix', input: '+123', expected: '+123' }, + { description: 'minus prefix', input: '-456', expected: '-456' }, + { description: 'at symbol prefix', input: '@lookup', expected: '@lookup' }, + { description: 'tab character prefix', input: '\ttabPrefix', expected: '\ttabPrefix' } + ])('should preserve raw characters when sanitizeFormulas is false, $description', ({ input, expected }) => { + expect(escapeCsvField(input, false)).toBe(expected); + }); +}); + +describe('formatCsv', () => { + it.each([ + { + description: 'standard headers and escaped row lines', + headers: ['col1', 'col2'], + rows: [ + ['val1', 'val2'], + ['val3,with,comma', 'val4 "quoted"'] + ], + expected: 'col1,col2\nval1,val2\n"val3,with,comma","val4 ""quoted"""\n' + }, + { + description: 'header with empty rows', + headers: ['header1'], + rows: [], + expected: 'header1\n' + }, + { + description: 'empty headers and empty rows', + headers: [], + rows: [], + expected: '\n' + }, + { + description: 'mixed data types (numbers, booleans, undefined)', + headers: ['id', 'name', 'score', 'active'], + rows: [ + [1, 'Alice', 98.5, true], + [2, 'Bob, Jr.', undefined, undefined] + ], + expected: 'id,name,score,active\n1,Alice,98.5,true\n2,"Bob, Jr.",,\n' + } + ])('should format CSV output correctly, $description', ({ headers, rows, expected }) => { + expect(formatCsv(headers, rows)).toBe(expected); + }); +}); + +describe('generateDiffCsv', () => { + it('should format all diff categories with status column prepended', () => { + const diff = { + added: [{ id: 'a1', val: 'Added Item' }], + removed: [{ id: 'r1', val: 'Removed Item' }], + modified: [{ id: 'm1', val: 'Modified Item' }], + unchanged: [{ id: 'u1', val: 'Unchanged Item' }] + }; + + const csv = generateDiffCsv(diff, { + headers: ['status', 'id', 'val'], + added: item => [item.id, item.val], + removed: item => [item.id, item.val], + modified: item => [item.id, item.val], + unchanged: item => [item.id, item.val] + }); + + const lines = csv.trim().split('\n'); + + expect(lines[0]).toBe('status,id,val'); + expect(lines[1]).toBe('ADDED,a1,Added Item'); + expect(lines[2]).toBe('REMOVED,r1,Removed Item'); + expect(lines[3]).toBe('MODIFIED,m1,Modified Item'); + expect(lines[4]).toBe('UNCHANGED,u1,Unchanged Item'); + }); +}); + +describe('saveCsvReport', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should recursively create directories, write CSV content to disk, and log completion', async () => { + mockMkdir.mockResolvedValue(undefined as never); + mockWriteFile.mockResolvedValue(undefined as never); + const mockLog = jest.spyOn(console, 'log').mockImplementation(() => {}); + + const targetPath = '/path/to/nested/reports/summary.csv'; + const csvContent = 'status,id,name\nADDED,1,Button\n'; + + await saveCsvReport(targetPath, csvContent); + + expect(mockMkdir).toHaveBeenCalledWith('/path/to/nested/reports', { recursive: true }); + expect(mockWriteFile).toHaveBeenCalledWith(targetPath, csvContent, 'utf-8'); + expect(mockLog).toHaveBeenCalledWith(`šŸ“„ Exported full CSV report: ${targetPath}`); + }); +}); diff --git a/tooling/collections/__tests__/helpers.test.ts b/tooling/collections/__tests__/helpers.test.ts new file mode 100644 index 00000000..c35944c4 --- /dev/null +++ b/tooling/collections/__tests__/helpers.test.ts @@ -0,0 +1,591 @@ +import * as childProcess from 'node:child_process'; +import { writeFile } from 'node:fs/promises'; +import { + extractCommitHash, + extractRepoInfo, + extractTrackedReposFromCatalog, + fetchCommitViaGitLsRemote, + fetchRepoCommit, + fetchLatestRepoHashes, + getReportsPath, + getSrcPath, + runUpdateTask, + verifyUrlReachability, + writeJsonCollection +} from '../helpers'; + +jest.mock('node:fs/promises', () => ({ + ...jest.requireActual('node:fs/promises'), + writeFile: jest.fn() +})); + +jest.mock('node:child_process', () => ({ + ...jest.requireActual('node:child_process'), + execSync: jest.fn() +})); + +const mockWriteFile = writeFile as jest.MockedFunction; + +afterEach(() => { + extractRepoInfo.memo.clear(); + fetchRepoCommit.memo.clear(); + verifyUrlReachability.memo.clear(); +}); + +describe('extractTrackedReposFromCatalog', () => { + it('should extract and deduplicate repositories across multiple categories', () => { + const mockCatalog = { + version: '1', + generated: '2026-01-01', + meta: { totalEntries: 2, totalDocs: 3, source: 'test' }, + docs: { + components: [ + { displayName: 'Button', pathSlug: 'button', path: 'https://raw.githubusercontent.com/patternfly/patternfly-react/sha1/pkg/button.md' }, + { displayName: 'Card', pathSlug: 'card', path: 'https://raw.githubusercontent.com/patternfly/patternfly-react/sha2/pkg/card.md' } + ], + guidelines: [ + { displayName: 'AI Guidelines', pathSlug: 'ai', path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/sha3/docs/ai.md' }, + { displayName: 'Invalid Path', pathSlug: 'invalid', path: 'https://example.com/invalid.md' } + ] + } + }; + + const result = extractTrackedReposFromCatalog(mockCatalog as any); + + expect(result).toHaveLength(2); + expect(result).toEqual([ + { owner: 'patternfly', repo: 'patternfly-react', branch: undefined }, + { owner: 'rh-uxd', repo: 'ai-helpers', branch: undefined } + ]); + }); + + it('should return an empty array when catalog contains no valid docs', () => { + expect(extractTrackedReposFromCatalog({} as any)).toEqual([]); + expect(extractTrackedReposFromCatalog({ docs: {} } as any)).toEqual([]); + }); + + it('should apply branch override when supplied', () => { + const catalog = { docs: { c: [{ path: 'https://raw.githubusercontent.com/org/repo/sha/file.md' }] } }; + const result = extractTrackedReposFromCatalog(catalog as any, 'develop'); + + expect(result[0]?.branch).toBe('develop'); + }); +}); + +describe('extractCommitHash', () => { + it.each([ + { + description: 'full 40-character SHA raw GitHub URL', + url: 'https://raw.githubusercontent.com/patternfly/patternfly-org/540bb0d31cb18670dd02857f80aa8b444fed9be9/packages/documentation-site/patternfly-docs/content/AI/ai.md', + expected: '540bb0d31cb18670dd02857f80aa8b444fed9be9' + }, + { + description: 'branch ref raw GitHub URL', + url: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/docs/components/data-display/table.md', + expected: 'main' + }, + { + description: 'non-raw GitHub URL', + url: 'https://github.com/patternfly/patternfly-org/blob/main/README.md', + expected: undefined + }, + { + description: 'non-GitHub URL', + url: 'https://example.com/invalid', + expected: undefined + }, + { + description: 'empty string URL', + url: '', + expected: undefined + }, + { + description: 'null input', + url: null as unknown as string, + expected: undefined + }, + { + description: 'undefined input', + url: undefined as unknown as string, + expected: undefined + } + ])('should extract commit hash or return undefined, $description', ({ url, expected }) => { + expect(extractCommitHash(url)).toBe(expected); + }); +}); + +describe('extractRepoInfo', () => { + it.each([ + { + description: 'valid raw GitHub URL with subpath', + url: 'https://raw.githubusercontent.com/patternfly/patternfly-org/540bb0d31359c381c8152331575ca2481e3fe1ff/packages/v4/src/content/components/button.md', + expected: { + owner: 'patternfly', + repo: 'patternfly-org', + ref: '540bb0d31359c381c8152331575ca2481e3fe1ff', + filePath: 'packages/v4/src/content/components/button.md' + } + }, + { + description: 'plain invalid URL string', + url: 'invalid-url', + expected: undefined + }, + { + description: 'incomplete GitHub path structure', + url: 'https://raw.githubusercontent.com/incomplete', + expected: undefined + }, + { + description: 'empty string input', + url: '', + expected: undefined + }, + { + description: 'null input', + url: null as unknown as string, + expected: undefined + }, + { + description: 'undefined input', + url: undefined as unknown as string, + expected: undefined + } + ])('should extract repository metadata or return undefined, $description', ({ url, expected }) => { + expect(extractRepoInfo(url)).toEqual(expected); + }); +}); + +describe('verifyUrlReachability', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should return true when HEAD request succeeds', async () => { + jest.spyOn(global, 'fetch').mockResolvedValue({ ok: true, status: 200 } as Response); + + const reachable = await verifyUrlReachability('https://raw.githubusercontent.com/test/file.md'); + + expect(reachable).toBe(true); + expect(global.fetch).toHaveBeenCalledWith( + 'https://raw.githubusercontent.com/test/file.md', + expect.objectContaining({ method: 'HEAD' }) + ); + }); + + it('should fall back to GET when HEAD returns 405 or 403', async () => { + jest + .spyOn(global, 'fetch') + .mockResolvedValueOnce({ ok: false, status: 405 } as Response) + .mockResolvedValueOnce({ ok: true, status: 200 } as Response); + + const reachable = await verifyUrlReachability('https://raw.githubusercontent.com/test/file.md'); + + expect(reachable).toBe(true); + expect(global.fetch).toHaveBeenCalledTimes(2); + expect(global.fetch).toHaveBeenLastCalledWith( + 'https://raw.githubusercontent.com/test/file.md', + expect.objectContaining({ method: 'GET' }) + ); + }); + + it('should return false when fetch throws network error or times out', async () => { + jest.spyOn(global, 'fetch').mockRejectedValue(new Error('Network error')); + + const reachable = await verifyUrlReachability('https://raw.githubusercontent.com/test/file.md'); + + expect(reachable).toBe(false); + }); +}); + +describe('fetchCommitViaGitLsRemote', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should return commit SHA when git ls-remote output contains SHA with HEAD', () => { + jest.spyOn(childProcess, 'execSync').mockReturnValue( + 'ref: refs/heads/main\tHEAD\n8f4a382e783457a4128509789234857234895723\tHEAD\n' + ); + + const sha = fetchCommitViaGitLsRemote('patternfly', 'patternfly-react'); + + expect(sha).toBe('8f4a382e783457a4128509789234857234895723'); + }); + + it('should fallback to 40-character SHA if symref line is missing', () => { + jest.spyOn(childProcess, 'execSync').mockReturnValue( + '8f4a382e783457a4128509789234857234895723\trefs/heads/main\n' + ); + + const sha = fetchCommitViaGitLsRemote('patternfly', 'patternfly-react'); + + expect(sha).toBe('8f4a382e783457a4128509789234857234895723'); + }); + + it('should return undefined if command fails or throws', () => { + jest.spyOn(childProcess, 'execSync').mockImplementation(() => { + throw new Error('Command failed'); + }); + + const sha = fetchCommitViaGitLsRemote('patternfly', 'patternfly-react'); + + expect(sha).toBeUndefined(); + }); +}); + +describe('fetchRepoCommit', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should resolve commit SHA for a specific branch from GitHub API', async () => { + jest.spyOn(global, 'fetch').mockResolvedValue({ + ok: true, + json: async () => ({ sha: 'mock-branch-sha' }) + } as Response); + + const sha = await fetchRepoCommit('patternfly', 'patternfly-react', 'main'); + + expect(sha).toBe('mock-branch-sha'); + expect(global.fetch).toHaveBeenCalledWith( + 'https://api.github.com/repos/patternfly/patternfly-react/commits/main', + expect.any(Object) + ); + }); + + it('should resolve commit SHA for default branch when branch is not specified', async () => { + jest.spyOn(global, 'fetch').mockResolvedValue({ + ok: true, + json: async () => [{ sha: 'mock-default-sha' }] + } as Response); + + const sha = await fetchRepoCommit('patternfly', 'patternfly-react'); + + expect(sha).toBe('mock-default-sha'); + expect(global.fetch).toHaveBeenCalledWith( + 'https://api.github.com/repos/patternfly/patternfly-react/commits', + expect.any(Object) + ); + }); + + it('should fallback to git ls-remote when GitHub API fails', async () => { + jest.spyOn(global, 'fetch').mockRejectedValue(new Error('Network error')); + jest.spyOn(childProcess, 'execSync').mockReturnValue( + '0123456789abcdef0123456789abcdef01234567\tHEAD\n' + ); + + const sha = await fetchRepoCommit('patternfly', 'patternfly-org'); + + expect(sha).toBe('0123456789abcdef0123456789abcdef01234567'); + }); + + it('should return undefined when API fails and git ls-remote fails', async () => { + jest.spyOn(global, 'fetch').mockRejectedValue(new Error('Network error')); + jest.spyOn(childProcess, 'execSync').mockImplementation(() => { + throw new Error('git error'); + }); + + const sha = await fetchRepoCommit('patternfly', 'patternfly-org'); + + expect(sha).toBeUndefined(); + }); +}); + +describe('fetchLatestRepoHashes', () => { + const originalEnv = process.env; + + beforeEach(() => { + process.env = { ...originalEnv }; + }); + + afterEach(() => { + process.env = originalEnv; + jest.restoreAllMocks(); + }); + + it('should fetch commit SHAs from GitHub API and index by owner/repo and repo name', async () => { + jest.spyOn(global, 'fetch').mockImplementation(async (input: RequestInfo | URL) => { + const url = typeof input === 'string' ? input : input.toString(); + + if (url.includes('patternfly-react')) { + return { + ok: true, + json: async () => ({ sha: 'mock-react-sha-123' }) + } as Response; + } + + return { ok: false, status: 404 } as Response; + }); + + const repos = [ + { owner: 'patternfly', repo: 'patternfly-react', branch: 'main' }, + { owner: 'patternfly', repo: 'nonexistent', branch: 'main' } + ]; + + const hashes = await fetchLatestRepoHashes(repos); + + expect(hashes.get('patternfly/patternfly-react')).toBe('mock-react-sha-123'); + expect(hashes.get('patternfly-react')).toBe('mock-react-sha-123'); + expect(hashes.has('patternfly/nonexistent')).toBe(false); + }); + + it('should query commits list when branch is not specified and handle array response', async () => { + let capturedUrl = ''; + + jest.spyOn(global, 'fetch').mockImplementation(async (input: RequestInfo | URL) => { + capturedUrl = typeof input === 'string' ? input : input.toString(); + + return { + ok: true, + json: async () => [{ sha: 'mock-default-sha-456' }] + } as Response; + }); + + const hashes = await fetchLatestRepoHashes([ + { owner: 'patternfly', repo: 'patternfly-elements' } + ]); + + expect(capturedUrl).toBe('https://api.github.com/repos/patternfly/patternfly-elements/commits'); + expect(hashes.get('patternfly/patternfly-elements')).toBe('mock-default-sha-456'); + expect(hashes.get('patternfly-elements')).toBe('mock-default-sha-456'); + }); + + it('should send standard unauthenticated headers with User-Agent and Accept without Authorization', async () => { + let capturedHeaders: Record = {}; + + jest.spyOn(global, 'fetch').mockImplementation(async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedHeaders = (init?.headers as Record) || {}; + + return { + ok: true, + json: async () => ({ sha: 'mock-public-sha' }) + } as Response; + }); + + await fetchLatestRepoHashes([ + { owner: 'patternfly', repo: 'patternfly-mcp', branch: 'main' } + ]); + + expect(capturedHeaders['User-Agent']).toBe('patternfly-mcp'); + expect(capturedHeaders.Accept).toBe('application/vnd.github.v3+json'); + expect(capturedHeaders.Authorization).toBeUndefined(); + }); + + it('should fall back to git ls-remote when GitHub API fails', async () => { + jest.spyOn(global, 'fetch').mockRejectedValue(new Error('Rate limit exceeded')); + jest.spyOn(childProcess, 'execSync').mockReturnValue( + '0123456789abcdef0123456789abcdef01234567\tHEAD\n' + ); + + const hashes = await fetchLatestRepoHashes([ + { owner: 'patternfly', repo: 'patternfly-org', branch: 'main' } + ]); + + expect(hashes.get('patternfly/patternfly-org')).toBe('0123456789abcdef0123456789abcdef01234567'); + }); + + it('should handle fetch failures gracefully without throwing', async () => { + jest.spyOn(global, 'fetch').mockRejectedValue(new Error('Rate limit exceeded')); + jest.spyOn(childProcess, 'execSync').mockImplementation(() => { + throw new Error('git command failed'); + }); + + const hashes = await fetchLatestRepoHashes([ + { owner: 'patternfly', repo: 'patternfly-org', branch: 'main' } + ]); + + expect(hashes.size).toBe(0); + }); +}); + +describe('writeJsonCollection', () => { + let logSpy: ReturnType; + + beforeEach(() => { + mockWriteFile.mockResolvedValue(undefined as never); + logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should format JSON as pretty-printed by default with trailing newline', async () => { + const targetPath = '/path/to/collection.json'; + const data = { key: 'value', count: 42 }; + + const result = await writeJsonCollection(targetPath, data); + + const expectedJson = JSON.stringify(data, null, 2); + + expect(mockWriteFile).toHaveBeenCalledWith(targetPath, expectedJson + '\n', 'utf-8'); + expect(result.jsonContent).toBe(expectedJson); + expect(logSpy).toHaveBeenCalledWith(`āœ… Updated ${targetPath}:`); + expect(logSpy).toHaveBeenCalledWith(expect.stringMatching(/File Size: \d+\.\d+ KB/)); + }); + + it('should format JSON in compact format when isPrettyPrint is false', async () => { + const targetPath = '/path/to/compact.json'; + const data = { a: 1, b: 2 }; + + const result = await writeJsonCollection(targetPath, data, { isPrettyPrint: false }); + + const expectedJson = JSON.stringify(data); + + expect(mockWriteFile).toHaveBeenCalledWith(targetPath, expectedJson + '\n', 'utf-8'); + expect(result.jsonContent).toBe(expectedJson); + }); + + it('should calculate elapsed time when startTime is supplied and log custom stats', async () => { + const targetPath = '/path/to/stats.json'; + const data = { items: [1, 2, 3] }; + const startTime = Date.now() - 1500; + + const result = await writeJsonCollection(targetPath, data, { + startTime, + stats: [ + { label: 'Total Items', value: 3 }, + { label: 'Status', value: 'OK' } + ] + }); + + expect(logSpy).toHaveBeenCalledWith(' - Total Items: 3'); + expect(logSpy).toHaveBeenCalledWith(' - Status: OK'); + expect(logSpy).toHaveBeenCalledWith(expect.stringMatching(/Time Elapsed: \d+\.\d+s/)); + expect(typeof result.durationSec).toBe('string'); + expect(typeof result.sizeKb).toBe('string'); + }); +}); + +describe('getSrcPath', () => { + it('should resolve default path containing the filename within src directory', () => { + const resolved = getSrcPath('test.json'); + + expect(resolved.endsWith('src/test.json')).toBe(true); + }); + + it('should support custom path overrides', () => { + const resolved = getSrcPath('custom.json', '../../custom'); + + expect(resolved.endsWith('custom/custom.json')).toBe(true); + }); +}); + +describe('getReportsPath', () => { + it('should be an aliased function', () => { + expect(typeof getReportsPath).toBe('function'); + }); +}); + +describe('runUpdateTask', () => { + const originalEnv = process.env; + + beforeEach(() => { + process.env = { ...originalEnv }; + jest.clearAllMocks(); + }); + + afterEach(() => { + process.env = originalEnv; + jest.restoreAllMocks(); + }); + + it.each([ + { + description: 'default env var unset', + env: {}, + options: undefined, + shouldInvoke: false + }, + { + description: 'default UPDATE_COLLECTIONS set to true', + env: { UPDATE_COLLECTIONS: 'true' }, + options: undefined, + shouldInvoke: true + }, + { + description: 'custom env variable set to true', + env: { CUSTOM_TRIGGER: 'true' }, + options: { envVar: 'CUSTOM_TRIGGER' }, + shouldInvoke: true + } + ])('should handle execution trigger condition, $description', async ({ env, options, shouldInvoke }) => { + Object.assign(process.env, env); + const taskFn = jest.fn().mockResolvedValue(undefined); + + await runUpdateTask('Test Task', taskFn, options); + + expect(taskFn).toHaveBeenCalledTimes(shouldInvoke ? 1 : 0); + }); + + it('should catch task errors, log failure message, and exit with status code 1', async () => { + process.env.UPDATE_COLLECTIONS = 'true'; + const testError = new Error('Task execution failed'); + const taskFn = jest.fn().mockRejectedValue(testError); + const errorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}); + const exitSpy = jest.spyOn(process, 'exit').mockImplementation((() => {}) as any); + + await runUpdateTask('Failing Task', taskFn); + + expect(errorSpy).toHaveBeenCalledWith('āŒ Failed to update Failing Task:', testError); + expect(exitSpy).toHaveBeenCalledWith(1); + }); +}); + +describe('memo cleanup and caching', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should allow clearing memo caches without errors', () => { + const url = 'https://raw.githubusercontent.com/patternfly/patternfly-react/main/packages/README.md'; + const first = extractRepoInfo.memo(url); + const second = extractRepoInfo.memo(url); + + expect(first).toEqual(second); + expect(extractRepoInfo.memo.clear()).toBe(true); + }); + + it('should cache fetchRepoCommit.memo calls and clear properly', async () => { + const fetchSpy = jest.spyOn(global, 'fetch').mockResolvedValue({ + ok: true, + json: async () => ({ sha: 'mock-memo-sha' }) + } as Response); + + const first = await fetchRepoCommit.memo('patternfly', 'patternfly-react', 'main'); + const second = await fetchRepoCommit.memo('patternfly', 'patternfly-react', 'main'); + + expect(first).toBe('mock-memo-sha'); + expect(second).toBe('mock-memo-sha'); + expect(fetchSpy).toHaveBeenCalledTimes(1); + + expect(fetchRepoCommit.memo.clear()).toBe(true); + + const third = await fetchRepoCommit.memo('patternfly', 'patternfly-react', 'main'); + + expect(third).toBe('mock-memo-sha'); + expect(fetchSpy).toHaveBeenCalledTimes(2); + }); + + it('should cache verifyUrlReachability.memo calls and clear properly', async () => { + const fetchSpy = jest.spyOn(global, 'fetch').mockResolvedValue({ + ok: true, + status: 200 + } as Response); + + const first = await verifyUrlReachability.memo('https://raw.githubusercontent.com/patternfly/test.md'); + const second = await verifyUrlReachability.memo('https://raw.githubusercontent.com/patternfly/test.md'); + + expect(first).toBe(true); + expect(second).toBe(true); + expect(fetchSpy).toHaveBeenCalledTimes(1); + + expect(verifyUrlReachability.memo.clear()).toBe(true); + + const third = await verifyUrlReachability.memo('https://raw.githubusercontent.com/patternfly/test.md'); + + expect(third).toBe(true); + expect(fetchSpy).toHaveBeenCalledTimes(2); + }); +}); diff --git a/tooling/collections/__tests__/summary.test.ts b/tooling/collections/__tests__/summary.test.ts new file mode 100644 index 00000000..8be5e646 --- /dev/null +++ b/tooling/collections/__tests__/summary.test.ts @@ -0,0 +1,114 @@ +import { jest } from '@jest/globals'; +import { printDiffSummary } from '../summary'; + +describe('printDiffSummary', () => { + let logSpy: ReturnType; + + beforeEach(() => { + logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should print no-change message when diff buckets are empty', () => { + printDiffSummary( + { added: [], removed: [], modified: [] }, + { + title: 'Empty Diff', + formatAdded: item => String(item), + formatRemoved: item => String(item), + formatModified: item => String(item) + } + ); + + expect(logSpy).toHaveBeenCalledWith('\nšŸ“Š Empty Diff:'); + expect(logSpy).toHaveBeenCalledWith(' ✨ No record additions, removals, or property modifications detected.'); + }); + + it('should print categorized changes with custom formatters', () => { + printDiffSummary( + { + added: [{ name: 'A1' }], + removed: [{ name: 'R1' }], + modified: [{ name: 'M1', reason: 'score' }] + }, + { + title: 'Custom Changes', + formatAdded: item => `Item: ${item.name}`, + formatRemoved: item => `Item: ${item.name}`, + formatModified: item => `Item: ${item.name} (${item.reason})` + } + ); + + expect(logSpy).toHaveBeenCalledWith('\nšŸ“Š Custom Changes:'); + expect(logSpy).toHaveBeenCalledWith(' āž• Added (1):'); + expect(logSpy).toHaveBeenCalledWith(' + Item: A1'); + expect(logSpy).toHaveBeenCalledWith(' āž– Removed (1):'); + expect(logSpy).toHaveBeenCalledWith(' - Item: R1'); + expect(logSpy).toHaveBeenCalledWith(' šŸ”„ Modified (1):'); + expect(logSpy).toHaveBeenCalledWith(' ~ Item: M1 (score)'); + }); + + it.each([ + { + description: 'default limits (10 added, 15 removed, 10 modified)', + diff: { + added: Array.from({ length: 12 }, (_, i) => `add-${i + 1}`), + removed: Array.from({ length: 18 }, (_, i) => `rem-${i + 1}`), + modified: Array.from({ length: 11 }, (_, i) => `mod-${i + 1}`) + }, + options: { + title: 'Truncated Diff', + formatAdded: (item: string) => item, + formatRemoved: (item: string) => item, + formatModified: (item: string) => item + }, + expectedLogs: [ + ' āž• Added (12):', + ' + add-10', + ' ... and 2 more', + ' āž– Removed (18):', + ' - rem-15', + ' ... and 3 more', + ' šŸ”„ Modified (11):', + ' ~ mod-10', + ' ... and 1 more' + ] + }, + { + description: 'custom limits (1 added, 1 removed, 2 modified)', + diff: { + added: ['a1', 'a2', 'a3'], + removed: ['r1', 'r2'], + modified: ['m1', 'm2', 'm3', 'm4'] + }, + options: { + title: 'Custom Limited Diff', + formatAdded: (item: string) => item, + formatRemoved: (item: string) => item, + formatModified: (item: string) => item, + limits: { added: 1, removed: 1, modified: 2 } + }, + expectedLogs: [ + ' āž• Added (3):', + ' + a1', + ' ... and 2 more', + ' āž– Removed (2):', + ' - r1', + ' ... and 1 more', + ' šŸ”„ Modified (4):', + ' ~ m1', + ' ~ m2', + ' ... and 2 more' + ] + } + ])('should truncate diff output properly, $description', ({ diff, options, expectedLogs }) => { + printDiffSummary(diff, options); + + for (const log of expectedLogs) { + expect(logSpy).toHaveBeenCalledWith(log); + } + }); +}); diff --git a/tests/scripts/update.collection.patternFlyApi.test.ts b/tooling/collections/__tests__/update.patternFlyApi.test.ts similarity index 74% rename from tests/scripts/update.collection.patternFlyApi.test.ts rename to tooling/collections/__tests__/update.patternFlyApi.test.ts index 3f5aaeda..e18ce7b8 100644 --- a/tests/scripts/update.collection.patternFlyApi.test.ts +++ b/tooling/collections/__tests__/update.patternFlyApi.test.ts @@ -1,9 +1,14 @@ import { readFileSync, existsSync } from 'node:fs'; -import { resolve } from 'node:path'; -import { expandApiEmbeddedCollection, type ApiEmbeddedCollection } from '../../src/collection.patternFlyApi'; -import { escapeCsvField, formatCsv, generateReportCsv, diffCollections, run } from '../../scripts/update.collection.patternFlyApi'; +import { expandApiEmbeddedCollection, type ApiEmbeddedCollection } from '../../../src/collection.patternFlyApi'; +import { + DEFAULT_API_FILENAME, + generateReportCsv, + diffCollections, + run +} from '../update.patternFlyApi'; +import { getSrcPath } from '../helpers'; -const COLLECTION_PATH = resolve(process.cwd(), 'src/collection.patternFlyApi.json'); +const COLLECTION_PATH = getSrcPath(DEFAULT_API_FILENAME); describe('collection.patternFlyApi', () => { it('should export the run function for programmatic invocation', () => { @@ -58,44 +63,6 @@ describe('collection.patternFlyApi', () => { }); describe('collection.patternFlyApi CSV Report Generator', () => { - it('should correctly escape fields with commas, quotes, and newlines', () => { - expect(escapeCsvField('normal')).toBe('normal'); - expect(escapeCsvField('with,comma')).toBe('"with,comma"'); - expect(escapeCsvField('with "quotes"')).toBe('"with ""quotes"""'); - expect(escapeCsvField('with\nnewline')).toBe('"with\nnewline"'); - expect(escapeCsvField(null)).toBe(''); - expect(escapeCsvField(undefined)).toBe(''); - expect(escapeCsvField(123)).toBe('123'); - }); - - it('should sanitize formula injection characters by default', () => { - expect(escapeCsvField('=SUM(1+1)')).toBe("'=SUM(1+1)"); - expect(escapeCsvField('+123')).toBe("'+123"); - expect(escapeCsvField('-456')).toBe("'-456"); - expect(escapeCsvField('@lookup')).toBe("'@lookup"); - expect(escapeCsvField('\ttabPrefix')).toBe("'\ttabPrefix"); - expect(escapeCsvField('\rreturnPrefix')).toBe('"\'\rreturnPrefix"'); - }); - - it('should preserve raw formula characters when sanitizeFormulas is set to false', () => { - expect(escapeCsvField('=SUM(1+1)', false)).toBe('=SUM(1+1)'); - expect(escapeCsvField('+123', false)).toBe('+123'); - expect(escapeCsvField('-456', false)).toBe('-456'); - expect(escapeCsvField('@lookup', false)).toBe('@lookup'); - }); - - it('should format header and row lines into standard CSV', () => { - const headers = ['col1', 'col2']; - const rows = [ - ['val1', 'val2'], - ['val3,with,comma', 'val4 "quoted"'] - ]; - - const result = formatCsv(headers, rows); - - expect(result).toBe('col1,col2\nval1,val2\n"val3,with,comma","val4 ""quoted"""\n'); - }); - it('should produce a full structured CSV report for added, removed, modified, and unchanged records', () => { const oldRecords = [ { p: 'endpoint/removed', n: 'Old Doc', d: 'Desc', c: 'text/html', q: 0.96 }, diff --git a/tooling/collections/__tests__/update.patternFlyDocs.test.ts b/tooling/collections/__tests__/update.patternFlyDocs.test.ts new file mode 100644 index 00000000..9d77cefd --- /dev/null +++ b/tooling/collections/__tests__/update.patternFlyDocs.test.ts @@ -0,0 +1,58 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { type PatternFlyMcpDocsCatalog } from '../../../src/docs.embedded'; +import { DEFAULT_DOCS_FILENAME, run } from '../update.patternFlyDocs'; +import { getSrcPath } from '../helpers'; + +const DOCS_PATH = getSrcPath(DEFAULT_DOCS_FILENAME); + +describe('collection.patternFlyDocs Script & Manifest Integrity', () => { + it('should export the run function for programmatic invocation', () => { + expect(typeof run).toBe('function'); + }); + + it('should have a generated documentation manifest file', () => { + expect(existsSync(DOCS_PATH)).toBe(true); + }); + + it('should have a consistent JSON schema and correct metadata counts', () => { + const raw = readFileSync(DOCS_PATH, 'utf-8'); + const catalog: PatternFlyMcpDocsCatalog = JSON.parse(raw); + + expect(catalog).toMatchObject({ + version: expect.any(String), + generated: expect.any(String), + meta: { + totalEntries: expect.any(Number), + totalDocs: expect.any(Number), + source: expect.any(String) + }, + docs: expect.any(Object) + }); + + const totalEntries = Object.keys(catalog.docs).length; + const totalDocs = Object.values(catalog.docs).reduce((acc, arr) => acc + arr.length, 0); + + expect(catalog.meta.totalEntries).toBe(totalEntries); + expect(catalog.meta.totalDocs).toBe(totalDocs); + }); + + it('should have records that are properly formatted with key properties', () => { + const raw = readFileSync(DOCS_PATH, 'utf-8'); + const catalog: PatternFlyMcpDocsCatalog = JSON.parse(raw); + const sample = Object.values(catalog.docs).flatMap(docs => docs).slice(0, 50); + + expect(sample.length).toBeGreaterThan(0); + for (const record of sample) { + expect(typeof record.displayName).toBe('string'); + expect(typeof record.description).toBe('string'); + expect(typeof record.pathSlug).toBe('string'); + expect(typeof record.section).toBe('string'); + expect(typeof record.category).toBe('string'); + expect(typeof record.source).toBe('string'); + expect(typeof record.path).toBe('string'); + expect(typeof record.version).toBe('string'); + + expect(record.path.startsWith('http')).toBe(true); + } + }); +}); diff --git a/tooling/collections/__tests__/update.patternFlyDocsHelpers.test.ts b/tooling/collections/__tests__/update.patternFlyDocsHelpers.test.ts new file mode 100644 index 00000000..47abf0c9 --- /dev/null +++ b/tooling/collections/__tests__/update.patternFlyDocsHelpers.test.ts @@ -0,0 +1,375 @@ +import { readFileSync, existsSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { type PatternFlyMcpDocsCatalog } from '../../../src/docs.embedded'; +import { + AI_GUIDELINE_ALIASES, + PINNED_HISTORICAL_REFS, + diffDocsManifests, + findApiRedundantDocs, + generateReportCsv, + recalculateManifestMetadata, + resolveApiEndpointForAiDoc +} from '../update.patternFlyDocsHelpers'; +import { DEFAULT_API_FILENAME, DEFAULT_DOCS_FILENAME } from '../update.patternFlyDocs'; +import { getSrcPath } from '../helpers'; + +const DOCS_PATH = getSrcPath(DEFAULT_DOCS_FILENAME); +const API_PATH = getSrcPath(DEFAULT_API_FILENAME); +const RECORDS_MAINT_DOCS_PATH = resolve(process.cwd(), `records-maint/${DEFAULT_DOCS_FILENAME}`); + +describe('docs.helpers CSV Report Generator', () => { + it('should produce a full structured CSV report for added, removed, modified, and unchanged doc entries', () => { + const diff = { + added: [ + { + category: 'NewCat', + record: { + displayName: 'New Doc', + description: 'New Description', + pathSlug: 'new-doc', + section: 'components', + category: 'react', + source: 'github', + path: 'https://raw.githubusercontent.com/patternfly/patternfly-react/831257aa7c49c3238e0f7afbb7cf219c62cd9e23/README.md', + version: 'v6' + }, + reason: 'new upstream document', + details: 'Discovered in manifest update' + } + ], + removed: [ + { + category: 'React', + record: { + displayName: 'Table Rules', + description: 'Old Table Rules', + pathSlug: 'table-rules', + section: 'components', + category: 'react', + source: 'github', + path: 'https://raw.githubusercontent.com/patternfly/ai-helpers/f7f8160c3f28b0bc7f64181d9466a425ac8329fc/docs/components/data-display/table.md', + version: 'v6' + }, + reason: 'superseded by API collection', + details: 'v6/AI/development-guidelines_table/text' + } + ], + modified: [ + { + category: 'Button', + record: { + displayName: 'Button Design', + description: 'Button design guidelines', + pathSlug: 'button-design', + section: 'components', + category: 'design-guidelines', + source: 'github', + path: 'https://raw.githubusercontent.com/patternfly/patternfly-org/540bb0d31cb18670dd02857f80aa8b444fed9be9/Button.md', + version: 'v6' + }, + reasons: ['hash update (9577561 -> 540bb0d)'], + previousHash: '957756128e8ddfc4be5db49e72312a2c43b9d220', + newHash: '540bb0d31cb18670dd02857f80aa8b444fed9be9' + } + ], + unchanged: [ + { + category: 'Alert', + record: { + displayName: 'Alert Design', + description: 'Alert design guidelines', + pathSlug: 'alert-design', + section: 'components', + category: 'design-guidelines', + source: 'github', + path: 'https://raw.githubusercontent.com/patternfly/patternfly-org/540bb0d31cb18670dd02857f80aa8b444fed9be9/Alert.md', + version: 'v6' + } + } + ] + }; + + const csv = generateReportCsv({ diff }); + const lines = csv.trim().split('\n'); + + expect(lines[0]).toBe('status,category,name,pathSlug,path,previousHash,newHash,reason,details'); + expect(lines.some(line => line.startsWith('ADDED,NewCat,New Doc') && line.includes('new upstream document'))).toBe(true); + expect(lines.some(line => line.startsWith('REMOVED,React,Table Rules') && line.includes('superseded by API collection'))).toBe(true); + expect(lines.some(line => line.startsWith('MODIFIED,Button,Button Design') && line.includes('hash update'))).toBe(true); + expect(lines.some(line => line.startsWith('UNCHANGED,Alert,Alert Design'))).toBe(true); + }); +}); + +describe('API Deduplication Litmus Test (Regression Guard)', () => { + it('correctly detects all 20 redundant AI-helper entries when cross-referenced with API seed', () => { + const sourcePath = existsSync(RECORDS_MAINT_DOCS_PATH) ? RECORDS_MAINT_DOCS_PATH : DOCS_PATH; + const rawDocs = readFileSync(sourcePath, 'utf-8'); + const rawApi = readFileSync(API_PATH, 'utf-8'); + const docs = JSON.parse(rawDocs); + const api = JSON.parse(rawApi); + + const redundant = findApiRedundantDocs(docs, api.records); + + if (docs.meta.totalDocs >= 341) { + expect(redundant.length).toBe(20); + + const expectedDisplayNames = [ + 'Table Rules', + 'React Charts', + 'React Chatbot', + 'AI Prompt Guidance', + 'Styling Standards', + 'PatternFly React Development Rules', + 'React Guidelines', + 'React Setup', + 'Development Environment', + 'Quick Start', + 'External Links', + 'React Troubleshooting', + 'Component Architecture', + 'Quick Deployment Guide for Prototypes', + 'Component Groups', + 'Data Display', + 'Layout Components' + ]; + const expectedPathSlugs = [ + 'ai-helpers-readme', + 'ai-helpers-contributing', + 'ai-helpers-contributing-skills' + ]; + + const redundantDisplayNames = new Set(redundant.map(report => report.record.displayName)); + const redundantPathSlugs = new Set(redundant.map(report => report.record.pathSlug)); + + for (const name of expectedDisplayNames) { + expect(redundantDisplayNames.has(name)).toBe(true); + } + for (const slug of expectedPathSlugs) { + expect(redundantPathSlugs.has(slug)).toBe(true); + } + } else { + expect(redundant.length).toBe(0); + } + }); + + it('preserves the 6 core root AIHelpers guides and ecosystem repositories without false positives', () => { + const rawDocs = readFileSync(DOCS_PATH, 'utf-8'); + const rawApi = readFileSync(API_PATH, 'utf-8'); + const docs = JSON.parse(rawDocs); + const api = JSON.parse(rawApi); + + const redundant = findApiRedundantDocs(docs, api.records); + const redundantPaths = new Set(redundant.map(report => report.record.path)); + + const rootAiHelpers = docs.docs['AIHelpers'] || []; + + for (const item of rootAiHelpers) { + if (item.pathSlug.startsWith('uxd-ai-helpers-')) { + expect(redundantPaths.has(item.path)).toBe(false); + } + } + + for (const entries of Object.values(docs.docs as Record)) { + for (const item of entries) { + if ( + item.path.includes('/patternfly-cli/') || + item.path.includes('/patternfly-elements/') || + item.path.includes('/patternfly-mcp/') || + item.path.includes('/pf-codemods/') + ) { + expect(redundantPaths.has(item.path)).toBe(false); + } + } + } + }); +}); + +describe('docs.helpers Manifest Recalculation & Diffing', () => { + it('should recalculate manifest metadata totalEntries and totalDocs accurately', () => { + const sampleCatalog: PatternFlyMcpDocsCatalog = { + version: '1', + generated: '2026-01-01T00:00:00.000Z', + meta: { + totalEntries: 2, + totalDocs: 3, + source: 'patternfly-mcp' + }, + docs: { + Button: [ + { + displayName: 'Button 1', + description: 'Desc 1', + pathSlug: 'button-1', + section: 'components', + category: 'react', + source: 'github', + path: 'https://raw.githubusercontent.com/patternfly/patternfly-react/1111111111111111111111111111111111111111/Button.md', + version: 'v6' + }, + { + displayName: 'Button 2', + description: 'Desc 2', + pathSlug: 'button-2', + section: 'components', + category: 'react', + source: 'github', + path: 'https://raw.githubusercontent.com/patternfly/patternfly-react/1111111111111111111111111111111111111111/Button2.md', + version: 'v6' + } + ], + Alert: [ + { + displayName: 'Alert 1', + description: 'Desc alert', + pathSlug: 'alert-1', + section: 'components', + category: 'react', + source: 'github', + path: 'https://raw.githubusercontent.com/patternfly/patternfly-org/2222222222222222222222222222222222222222/Alert.md', + version: 'v6' + } + ] + } + }; + + const latestHashes = new Map([ + ['patternfly/patternfly-react', '3333333333333333333333333333333333333333'] + ]); + + const updated = recalculateManifestMetadata(sampleCatalog, { latestHashes }); + + expect(updated.meta.totalEntries).toBe(2); + expect(updated.meta.totalDocs).toBe(3); + expect(updated.docs.Button?.[0]?.path).toContain('3333333333333333333333333333333333333333'); + expect(updated.docs.Alert?.[0]?.path).toContain('2222222222222222222222222222222222222222'); + }); + + it('should diff manifests and categorize added, removed, modified, and unchanged entries', () => { + const oldCatalog: PatternFlyMcpDocsCatalog = { + meta: { totalEntries: 2, totalDocs: 2, source: 'test' }, + docs: { + Cat1: [ + { + displayName: 'Doc A', + description: 'Desc A', + pathSlug: 'doc-a', + section: 'sec', + category: 'cat', + source: 'github', + path: 'https://raw.githubusercontent.com/owner/repo/hash1111111111111111111111111111111111111111/a.md', + version: 'v6' + } + ], + Cat2: [ + { + displayName: 'Doc B', + description: 'Desc B', + pathSlug: 'doc-b', + section: 'sec', + category: 'cat', + source: 'github', + path: 'https://raw.githubusercontent.com/owner/repo/hash1111111111111111111111111111111111111111/b.md', + version: 'v6' + } + ] + } + }; + + const newCatalog: PatternFlyMcpDocsCatalog = { + meta: { totalEntries: 2, totalDocs: 2, source: 'test' }, + docs: { + Cat1: [ + { + displayName: 'Doc A', + description: 'Desc A', + pathSlug: 'doc-a', + section: 'sec', + category: 'cat', + source: 'github', + path: 'https://raw.githubusercontent.com/owner/repo/hash2222222222222222222222222222222222222222/a.md', + version: 'v6' + } + ], + Cat3: [ + { + displayName: 'Doc C', + description: 'Desc C', + pathSlug: 'doc-c', + section: 'sec', + category: 'cat', + source: 'github', + path: 'https://raw.githubusercontent.com/owner/repo/hash2222222222222222222222222222222222222222/c.md', + version: 'v6' + } + ] + } + }; + + const diff = diffDocsManifests(oldCatalog, newCatalog); + + expect(diff.modified.length).toBe(1); + expect(diff.modified[0]?.record.displayName).toBe('Doc A'); + expect(diff.removed.length).toBe(1); + expect(diff.removed[0]?.record.displayName).toBe('Doc B'); + expect(diff.added.length).toBe(1); + expect(diff.added[0]?.record.displayName).toBe('Doc C'); + }); + + it('should maintain declarative pinned historical hashes and guideline aliases', () => { + expect(PINNED_HISTORICAL_REFS instanceof Set).toBe(true); + expect(PINNED_HISTORICAL_REFS.has('ec02b437ec72b6e4cc4e28524516288f4acf9fdf')).toBe(true); + expect(PINNED_HISTORICAL_REFS.has('e8cca17430a8ccb062ed1878073165417a081b34')).toBe(true); + + expect(AI_GUIDELINE_ALIASES['table-rules']).toBe('table'); + expect(AI_GUIDELINE_ALIASES['layout-components']).toBe('layout'); + expect(AI_GUIDELINE_ALIASES['development-rules']).toBe('overview'); + }); + + const highQualityApi = [ + { p: 'v6/AI/ai-assisted-development_marketplace/text', q: 1 }, + { p: 'v6/AI/development-guidelines_overview/text', q: 1 }, + { p: 'v6/AI/development-guidelines_table/text', q: 1 }, + { p: 'v6/AI/development-guidelines_charts/text', q: 1 } + ]; + + it.each([ + { + description: 'README alias matching marketplace endpoint', + doc: { + displayName: 'AI Helpers README', + pathSlug: 'ai-helpers-readme', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/README.md' + }, + expected: 'v6/AI/ai-assisted-development_marketplace/text' + }, + { + description: 'table rules alias matching table endpoint', + doc: { + displayName: 'Table Rules', + pathSlug: 'table-rules', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/docs/components/data-display/table.md' + }, + expected: 'v6/AI/development-guidelines_table/text' + }, + { + description: 'charts token matching charts endpoint', + doc: { + displayName: 'Charts Rules', + pathSlug: 'charts', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/docs/charts/README.md' + }, + expected: 'v6/AI/development-guidelines_charts/text' + } + ])('should resolve AI endpoint, $description', ({ doc, expected }) => { + const fullDoc = { + version: 'v6', + category: 'AI', + description: '', + section: '', + source: 'github' as const, + ...doc + }; + + expect(resolveApiEndpointForAiDoc(fullDoc, highQualityApi)).toBe(expected); + }); +}); diff --git a/tooling/collections/csv.ts b/tooling/collections/csv.ts new file mode 100644 index 00000000..99f67789 --- /dev/null +++ b/tooling/collections/csv.ts @@ -0,0 +1,172 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname } from 'node:path'; +import { getReportsPath } from './helpers'; + +/** + * Value types stored in a CSV cell. + * + * - `string`: For text-based values within a CSV cell. + * - `number`: For numeric values within a CSV cell. + * - `boolean`: For true/false values within a CSV cell. + * - `undefined`: Represents the absence of a value within a CSV cell. + */ +type CsvCellValue = string | number | boolean | undefined; + +/** + * Diff status + * + * - `ADDED`: Item exists in the target but not in the source. + * - `REMOVED`: Item exists in the source but not in the target. + * - `MODIFIED`: Item exists in both the source and target but has changed. + * - `UNCHANGED`: Item exists in both the source and the target without any change. + */ +type DiffStatus = 'ADDED' | 'REMOVED' | 'MODIFIED' | 'UNCHANGED'; + +/** + * Diff row mapping transformations for each status category. + */ +interface DiffReportRowMappers { + headers: string[]; + added: (item: TAdded) => CsvCellValue[]; + removed: (item: TRemoved) => CsvCellValue[]; + modified: (item: TModified) => CsvCellValue[]; + unchanged: (item: TUnchanged) => CsvCellValue[]; +} + +/** + * Generic container for categorized diff sets. + */ +interface GenericDiff { + added: Iterable; + removed: Iterable; + modified: Iterable; + unchanged: Iterable; +} + +/** + * Generate a YYYYMMDD date prefix with trailing hyphen for report filenames. + * + * @param [date=new Date()] - Target date to format + * @returns Formatted date prefix (e.g., "20261009-") + */ +const getReportDatePrefix = (date = new Date()): string => { + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); + const day = String(date.getDate()).padStart(2, '0'); + + return `${year}${month}${day}-`; +}; + +/** + * Resolve the default absolute path for a CSV report with a YYYYMMDD- prefix. + * + * @param filename - Base report filename + * @param [date=new Date()] - Date to use for the timestamp prefix + * @returns Fully resolved filesystem path in the reports directory + */ +const getDefaultReportPath = (filename: string, date = new Date()): string => { + const prefix = getReportDatePrefix(date); + + return getReportsPath(`${prefix}${filename}`); +}; + +/** + * Safely escape and format a field for standard RFC 4180 CSV output. + * + * @note **CSV / Formula Injection:** By default (`sanitizeFormulas = true`), leading formula + * trigger characters are prefixed with a single quote to prevent spreadsheet execution. Pass + * `false` to preserve strict raw string fidelity for automated downstream parsers. + * + * @param field - Value to format for CSV + * @param [sanitizeFormulas=true] - Whether to prefix formula trigger characters with a single quote + * @returns RFC 4180 compliant CSV cell string + */ +const escapeCsvField = (field: unknown, sanitizeFormulas = true): string => { + if (field === null || field === undefined) { + return ''; + } + + let str = String(field); + + if (sanitizeFormulas && /^[=+\-@\t\r]/.test(str)) { + str = `'${str}`; + } + + if (str.includes(',') || str.includes('"') || str.includes('\n') || str.includes('\r')) { + return `"${str.replace(/"/g, '""')}"`; + } + + return str; +}; + +/** + * Format rows and headers into standard CSV string. + * + * @param headers - Column headers + * @param rows - Table rows + * @returns Formatted RFC 4180 CSV string + */ +const formatCsv = ( + headers: string[], + rows: CsvCellValue[][] +): string => { + const headerLine = headers.map(header => escapeCsvField(header)).join(','); + const rowLines = rows.map(row => row.map(cell => escapeCsvField(cell)).join(',')); + + return [headerLine, ...rowLines].join('\n') + '\n'; +}; + +/** + * Generic CSV generator for collection diff reports. + * Handles iteration across status buckets, prepends status tags, and formats RFC 4180 CSV. + * + * @param diff - Categorized diff buckets + * @param mappers - Column headers and row mapping functions + * @returns RFC 4180 CSV string + */ +const generateDiffCsv = ( + diff: GenericDiff, + mappers: DiffReportRowMappers +): string => { + const rows: CsvCellValue[][] = []; + + for (const item of diff.added) { + rows.push(['ADDED', ...mappers.added(item)]); + } + for (const item of diff.removed) { + rows.push(['REMOVED', ...mappers.removed(item)]); + } + for (const item of diff.modified) { + rows.push(['MODIFIED', ...mappers.modified(item)]); + } + for (const item of diff.unchanged) { + rows.push(['UNCHANGED', ...mappers.unchanged(item)]); + } + + return formatCsv(mappers.headers, rows); +}; + +/** + * Standardized helper to persist a CSV report to disk and log output status. + * + * @param targetPath - Absolute or relative file path for the CSV report + * @param csvContent - Pre-formatted CSV string content + */ +const saveCsvReport = async (targetPath: string, csvContent: string): Promise => { + await mkdir(dirname(targetPath), { recursive: true }); + await writeFile(targetPath, csvContent, 'utf-8'); + console.log(`šŸ“„ Exported full CSV report: ${targetPath}`); +}; + +export { + getDefaultReportPath, + getReportDatePrefix, + escapeCsvField, + formatCsv, + generateDiffCsv, + saveCsvReport, + type CsvCellValue, + type DiffReportRowMappers, + type DiffStatus, + type GenericDiff +}; diff --git a/tooling/collections/helpers.ts b/tooling/collections/helpers.ts new file mode 100644 index 00000000..e10a87a2 --- /dev/null +++ b/tooling/collections/helpers.ts @@ -0,0 +1,380 @@ +import { execSync } from 'node:child_process'; +import { writeFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { memo } from '../../src/server.caching'; +import { type PatternFlyMcpDocsCatalog } from '../../src/docs.embedded'; + +/** + * Information extracted from a raw GitHub documentation URL. + */ +interface GitHubUrlInfo { + owner: string; + repo: string; + ref: string; + filePath: string; +} + +/** + * Repository tracking definition for upstream release queries. + */ +interface TrackedRepository { + owner: string; + repo: string; + branch?: string | undefined; +} + +/** + * Extract commit hash or ref from a raw GitHub documentation URL. + * + * @param url - Raw GitHub URL + * @returns Hash/ref string or undefined if not a recognized GitHub raw URL + */ +const extractCommitHash = (url: string): string | undefined => { + if (!url || typeof url !== 'string') { + return undefined; + } + + const match = url.match(/^https:\/\/raw\.githubusercontent\.com\/[^/]+\/[^/]+\/([^/]+)\//); + + return match && match[1] ? match[1] : undefined; +}; + +/** + * Extract structured repository and path information from a raw GitHub URL. + * + * @param url - Raw GitHub URL + * @returns GitHubUrlInfo or undefined if not a recognized raw URL + */ +const extractRepoInfo = (url: string): GitHubUrlInfo | undefined => { + if (!url || typeof url !== 'string') { + return undefined; + } + + const match = url.match(/^https:\/\/raw\.githubusercontent\.com\/([^/]+)\/([^/]+)\/([^/]+)\/(.+)$/); + + if (!match || !match[1] || !match[2] || !match[3] || !match[4]) { + return undefined; + } + + return { + owner: match[1], + repo: match[2], + ref: match[3], + filePath: match[4] + }; +}; + +/** + * Memoized version of extractRepoInfo. + */ +extractRepoInfo.memo = memo(extractRepoInfo, { + cacheLimit: 500, + keyHash: args => args[0] +}); + +/** + * Dynamically extract unique repositories referenced across all documents in a catalog. + * + * @param catalog - Documentation catalog to inspect + * @param [defaultBranch] - Optional explicit tracking branch override + * @returns Array of unique TrackedRepository definitions + */ +const extractTrackedReposFromCatalog = ( + catalog: PatternFlyMcpDocsCatalog, + defaultBranch?: string +): TrackedRepository[] => { + const uniqueRepos = new Map(); + + for (const entries of Object.values(catalog?.docs || {})) { + for (const doc of entries) { + if (!doc?.path || typeof doc.path !== 'string') { + continue; + } + + const info = extractRepoInfo(doc.path); + + if (info && info.owner && info.repo) { + const key = `${info.owner}/${info.repo}`.toLowerCase(); + + if (!uniqueRepos.has(key)) { + uniqueRepos.set(key, { owner: info.owner, repo: info.repo, branch: defaultBranch }); + } + } + } + } + + return Array.from(uniqueRepos.values()); +}; + +/** + * Fallback helper to query HEAD commit SHA using `git ls-remote --symref`. + * + * @param owner - Repository owner/organization + * @param repo - Repository name + * @returns Commit SHA or undefined if git is unavailable or command fails + */ +const fetchCommitViaGitLsRemote = (owner: string, repo: string): string | undefined => { + try { + const remoteUrl = `https://github.com/${owner}/${repo}.git`; + const output = execSync(`git ls-remote --symref ${remoteUrl} HEAD`, { + encoding: 'utf-8', + timeout: 10_000, + stdio: ['ignore', 'pipe', 'ignore'] + }); + + const shaMatch = output.match(/^([0-9a-f]{40})\s+HEAD/m); + + if (shaMatch && shaMatch[1]) { + return shaMatch[1]; + } + + const fallbackMatch = output.match(/([0-9a-f]{40})/); + + return fallbackMatch && fallbackMatch[1] ? fallbackMatch[1] : undefined; + } catch { + return undefined; + } +}; + +/** + * Internal helper to perform a probe request with guaranteed timer cleanup. + * + * @param url - Target URL to probe + * @param method - HTTP method to use ('HEAD' | 'GET') + * @param timeoutMs - Timeout in milliseconds + * @returns Promise resolving to Response + */ +const probeUrl = async (url: string, method: 'HEAD' | 'GET', timeoutMs: number): Promise => { + const controller = new AbortController(); + const timeoutId = setTimeout(() => { + controller.abort(); + }, timeoutMs); + + try { + return await fetch(url, { + method, + headers: { 'User-Agent': 'patternfly-mcp-audit' }, + signal: controller.signal + }); + } finally { + clearTimeout(timeoutId); + } +}; + +/** + * Probe URL reachability using HTTP HEAD / GET request. + * + * @param url - Target URL to probe + * @param [timeoutMs=5000] - Timeout in milliseconds + * @returns Promise resolving to true if status is 2xx, false otherwise + */ +const verifyUrlReachability = async (url: string, timeoutMs = 5000): Promise => { + try { + const response = await probeUrl(url, 'HEAD', timeoutMs); + + if (response.ok) { + return true; + } + + // Fallback to GET for hosts that reject HEAD requests + if (response.status === 405 || response.status === 403) { + const getResponse = await probeUrl(url, 'GET', timeoutMs); + + return getResponse.ok; + } + + return false; + } catch { + return false; + } +}; + +/** + * Memoized version of verifyUrlReachability. + */ +verifyUrlReachability.memo = memo(verifyUrlReachability, { + cacheLimit: 200, + keyHash: args => `${args[0]}:${args[1] || 5000}` +}); + +/** + * Resolve latest commit SHA for an individual repository via GitHub API or git ls-remote fallback. + * + * @param owner - Repository owner + * @param repo - Repository name + * @param [branch] - Optional target branch + * @returns Commit SHA or undefined if unresolved + */ +const fetchRepoCommit = async ( + owner: string, + repo: string, + branch?: string +): Promise => { + let sha: string | undefined = undefined; + + try { + const url = branch + ? `https://api.github.com/repos/${owner}/${repo}/commits/${branch}` + : `https://api.github.com/repos/${owner}/${repo}/commits`; + + const response = await fetch(url, { + headers: { + 'User-Agent': 'patternfly-mcp', + Accept: 'application/vnd.github.v3+json' + } + }); + + if (response.ok) { + const data = await response.json(); + + if (Array.isArray(data) && data[0]?.sha) { + sha = data[0].sha; + } else if (data && typeof data === 'object' && 'sha' in data && typeof data.sha === 'string') { + sha = data.sha; + } + } + } catch { + // Fallback on network/API failure + } + + return sha || fetchCommitViaGitLsRemote(owner, repo); +}; + +/** + * Memoized version of fetchRepoCommit. + */ +fetchRepoCommit.memo = memo(fetchRepoCommit, { + cacheLimit: 50, + expire: 60_000, + keyHash: args => `${args[0]}/${args[1]}:${args[2] || 'default'}` +}); + +/** + * Fetch latest commit hashes for tracked repositories via GitHub API with git ls-remote fallback. + * + * @param repos - Repositories to query + * @returns Map of "owner/repo" and "repo" to commit SHA + */ +const fetchLatestRepoHashes = async ( + repos: TrackedRepository[] = [] +): Promise> => { + const hashes = new Map(); + + for (const { owner, repo, branch } of repos) { + const key = `${owner}/${repo}`; + const sha = await fetchRepoCommit.memo(owner, repo, branch); + + if (sha) { + hashes.set(key, sha); + hashes.set(repo, sha); + } + } + + return hashes; +}; + +/** + * Options for writeJsonCollection. + */ +interface WriteJsonCollectionOptions { + isPrettyPrint?: boolean; + startTime?: number; + stats?: Array<{ label: string; value: string | number }>; +} + +/** + * Serialize data to JSON, write to target path, and log file size and elapsed time. + * + * @param targetPath - Absolute or relative file path for JSON output + * @param data - Data payload to serialize + * @param [options={}] - Output and logging options + * @returns Serialized JSON content and formatted metrics + */ +const writeJsonCollection = async ( + targetPath: string, + data: T, + options: WriteJsonCollectionOptions = {} +): Promise<{ jsonContent: string; sizeKb: string; durationSec: string }> => { + const isPretty = options.isPrettyPrint !== false; + const jsonContent = isPretty ? JSON.stringify(data, null, 2) : JSON.stringify(data); + + await writeFile(targetPath, jsonContent + '\n', 'utf-8'); + + const durationSec = options.startTime ? ((Date.now() - options.startTime) / 1000).toFixed(1) : '0.0'; + const sizeKb = (Buffer.byteLength(jsonContent, 'utf-8') / 1024).toFixed(1); + + console.log(`āœ… Updated ${targetPath}:`); + if (options.stats) { + for (const { label, value } of options.stats) { + console.log(` - ${label}: ${value}`); + } + } + console.log(` - File Size: ${sizeKb} KB`); + if (options.startTime) { + console.log(` - Time Elapsed: ${durationSec}s`); + } + + return { jsonContent, sizeKb, durationSec }; +}; + +/** + * Resolve the default absolute path for a JSON collection file in the src directory. + * + * @param filename - Collection JSON filename + * @param [path] - Default optional path to prepend to the filename. + * @returns Fully resolved filesystem path in the src directory + */ +const getSrcPath = (filename: string, path: string = '../../src'): string => + resolve(fileURLToPath(new URL(`${path}/${filename}`, import.meta.url))); + +/** + * Resolve the default absolute path for a report file in the reports directory. + * + * @param filename - Report filename + * @param [path] - Default optional path to prepend to the filename. + * @returns Fully resolved filesystem path in the reports directory + */ +const getReportsPath = (filename: string, path: string = '../../reports'): string => + getSrcPath(filename, path); + +/** + * Generic lifecycle runner for collection update tasks invoked via environment variables. + * + * @param taskName - Human-readable name of the task for logging + * @param taskFn - Async task function to execute + * @param [options={}] - Execution options + * @param [options.envVar='UPDATE_COLLECTIONS'] - Environment variable trigger name + */ +const runUpdateTask = async ( + taskName: string, + taskFn: () => Promise, + options: { envVar?: string } = { envVar: 'UPDATE_COLLECTIONS' } +): Promise => { + if (process.env[options.envVar || 'UPDATE_COLLECTIONS'] === 'true') { + try { + await taskFn(); + } catch (error) { + console.error(`āŒ Failed to update ${taskName}:`, error); + process.exit(1); + } + } +}; + +export { + extractCommitHash, + extractRepoInfo, + extractTrackedReposFromCatalog, + fetchCommitViaGitLsRemote, + fetchRepoCommit, + fetchLatestRepoHashes, + getReportsPath, + getSrcPath, + probeUrl, + runUpdateTask, + verifyUrlReachability, + writeJsonCollection, + type GitHubUrlInfo, + type TrackedRepository, + type WriteJsonCollectionOptions +}; diff --git a/tooling/collections/summary.ts b/tooling/collections/summary.ts new file mode 100644 index 00000000..5e9b095d --- /dev/null +++ b/tooling/collections/summary.ts @@ -0,0 +1,101 @@ +/** + * Generic container for categorized diff items. + */ +interface DiffSummaryBuckets { + added: TAdded[]; + removed: TRemoved[]; + modified: TModified[]; +} + +/** + * Truncation thresholds for console diff output. + */ +interface DiffSummaryLimits { + added?: number; + removed?: number; + modified?: number; +} + +/** + * Formatters and configuration for diff summary output. + */ +interface DiffSummaryFormatters { + title: string; + formatAdded: (item: TAdded) => string; + formatRemoved: (item: TRemoved) => string; + formatModified: (item: TModified) => string; + limits?: DiffSummaryLimits; +} + +/** + * Print a truncated, categorized summary of additions, removals, and modifications to the console. + * + * @param diff - Categorized diff items + * @param options - Formatters and configuration object + * @param options.title - Title for the summary report + * @param options.formatAdded - Formatter function for added items + * @param options.formatRemoved - Formatter function for removed items + * @param options.formatModified - Formatter function for modified items + * @param [options.limits] - Truncation limits + */ +const printDiffSummary = ( + diff: DiffSummaryBuckets, + { + title, + formatAdded, + formatRemoved, + formatModified, + limits = {} + }: DiffSummaryFormatters +): void => { + const { added, removed, modified } = diff; + const hasChanges = added.length > 0 || removed.length > 0 || modified.length > 0; + const maxAdded = limits.added ?? 10; + const maxRemoved = limits.removed ?? 15; + const maxModified = limits.modified ?? 10; + + console.log(`\nšŸ“Š ${title}:`); + + if (!hasChanges) { + console.log(' ✨ No record additions, removals, or property modifications detected.'); + + return; + } + + if (added.length > 0) { + console.log(` āž• Added (${added.length}):`); + added.slice(0, maxAdded).forEach(item => { + console.log(` + ${formatAdded(item)}`); + }); + if (added.length > maxAdded) { + console.log(` ... and ${added.length - maxAdded} more`); + } + } + + if (removed.length > 0) { + console.log(` āž– Removed (${removed.length}):`); + removed.slice(0, maxRemoved).forEach(item => { + console.log(` - ${formatRemoved(item)}`); + }); + if (removed.length > maxRemoved) { + console.log(` ... and ${removed.length - maxRemoved} more`); + } + } + + if (modified.length > 0) { + console.log(` šŸ”„ Modified (${modified.length}):`); + modified.slice(0, maxModified).forEach(item => { + console.log(` ~ ${formatModified(item)}`); + }); + if (modified.length > maxModified) { + console.log(` ... and ${modified.length - maxModified} more`); + } + } +}; + +export { + printDiffSummary, + type DiffSummaryBuckets, + type DiffSummaryFormatters, + type DiffSummaryLimits +}; diff --git a/scripts/update.collection.patternFlyApi.ts b/tooling/collections/update.patternFlyApi.ts similarity index 55% rename from scripts/update.collection.patternFlyApi.ts rename to tooling/collections/update.patternFlyApi.ts index ab9f5add..bcdc0717 100644 --- a/scripts/update.collection.patternFlyApi.ts +++ b/tooling/collections/update.patternFlyApi.ts @@ -1,6 +1,4 @@ -import { readFile, writeFile, mkdir } from 'node:fs/promises'; -import { dirname, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; +import { readFile } from 'node:fs/promises'; import { apiSpider, contentMetadata, @@ -9,10 +7,16 @@ import { type ApiEmbedded, type ApiEmbeddedCollection, MIN_API_QUALITY_THRESHOLD -} from '../src/collection.patternFlyApi'; -import { getSessionOptions, getOptions, runWithOptions } from '../src/options.context'; -import { createLogger } from '../src/logger'; -import { type LoggingSession } from '../src/options.defaults'; +} from '../../src/collection.patternFlyApi'; +import { getLoggerOptions, getOptions, runWithOptions } from '../../src/options.context'; +import { createLogger } from '../../src/logger'; +import { + generateDiffCsv, + getDefaultReportPath, + saveCsvReport +} from './csv'; +import { getSrcPath, runUpdateTask, writeJsonCollection } from './helpers'; +import { printDiffSummary } from './summary'; /** * Reason classification for omitted or removed API records. @@ -52,46 +56,27 @@ interface GenerateCsvReportOptions { } /** - * Safely escape and format a field for standard RFC 4180 CSV output. - * - * @note **CSV / Formula Injection:** By default (`sanitizeFormulas = true`), leading formula - * trigger characters are prefixed with a single quote to prevent spreadsheet execution. Pass - * `false` to preserve strict raw string fidelity for automated downstream parsers. - * - * @param field - Value to format for CSV - * @param [sanitizeFormulas=true] - Whether to prefix formula trigger characters with a single quote - * @returns RFC 4180 compliant CSV cell string + * Configurable default filename for the API collection. */ -const escapeCsvField = (field: unknown, sanitizeFormulas = true): string => { - if (field === null || field === undefined) { - return ''; - } - - let str = String(field); - - if (sanitizeFormulas && /^[=+\-@\t\r]/.test(str)) { - str = `'${str}`; - } - - if (str.includes(',') || str.includes('"') || str.includes('\n') || str.includes('\r')) { - return `"${str.replace(/"/g, '""')}"`; - } - - return str; -}; +const DEFAULT_API_FILENAME = 'collection.patternFlyApi.json'; /** - * Format rows and headers into standard CSV string. - * - * @param headers - Column headers - * @param rows - Table rows + * Configurable default filename for the API report. */ -const formatCsv = (headers: string[], rows: (string | number | undefined | null)[][]): string => { - const headerLine = headers.map(field => escapeCsvField(field)).join(','); - const rowLines = rows.map(row => row.map(cell => escapeCsvField(cell)).join(',')); +const DEFAULT_API_REPORT_FILENAME = 'collection.patternFlyApi.report.csv'; - return [headerLine, ...rowLines].join('\n') + '\n'; -}; +/** + * Options for running the PatternFly API embedded collection update. + */ +interface UpdateApiOptions { + filename?: string | undefined; + reportFilename?: string | undefined; + outputPath?: string | undefined; + csvOutputPath?: string | undefined; + isPrettyPrint?: boolean | undefined; + filterLowQualityRecords?: boolean | undefined; + outputCsv?: boolean | undefined; +} /** * Generate a complete, non-truncated CSV report for additions, removals, modifications, and unchanged records. @@ -109,38 +94,39 @@ const generateReportCsv = ({ crawledMap }: GenerateCsvReportOptions): string => { const oldMap = new Map(oldRecords.map(record => [record.p, record])); - const headers = ['status', 'path', 'name', 'previousQualityScore', 'newQualityScore', 'contentType', 'reason', 'details']; - const rows: (string | number | undefined | null)[][] = []; - - for (const record of diff.added) { - rows.push(['ADDED', record.p, record.n, '', record.q, record.c, '', '']); - } - - for (const { record, reason, details } of diff.removed) { - const newQualityScore = crawledMap.get(record.p)?.entry.qualityScore ?? ''; - - rows.push(['REMOVED', record.p, record.n, record.q, newQualityScore, record.c, reason, details || '']); - } - - for (const { record, reasons } of diff.modified) { - const previousQualityScore = oldMap.get(record.p)?.q ?? ''; - - rows.push(['MODIFIED', record.p, record.n, previousQualityScore, record.q, record.c, 'property changes', reasons.join('; ')]); - } - const changedPaths = new Set([ ...diff.added.map(record => record.p), ...diff.removed.map(removedItem => removedItem.record.p), ...diff.modified.map(modifiedItem => modifiedItem.record.p) ]); - - for (const record of newRecords) { - if (!changedPaths.has(record.p)) { - rows.push(['UNCHANGED', record.p, record.n, record.q, record.q, record.c, '', '']); + const unchanged = newRecords.filter(record => !changedPaths.has(record.p)); + + return generateDiffCsv( + { ...diff, unchanged }, + { + headers: ['status', 'path', 'name', 'previousQualityScore', 'newQualityScore', 'contentType', 'reason', 'details'], + added: record => [record.p, record.n, '', record.q, record.c, '', ''], + removed: ({ record, reason, details }) => [ + record.p, + record.n, + record.q, + crawledMap.get(record.p)?.entry.qualityScore ?? '', + record.c, + reason, + details || '' + ], + modified: ({ record, reasons }) => [ + record.p, + record.n, + oldMap.get(record.p)?.q ?? '', + record.q, + record.c, + 'property changes', + reasons.join('; ') + ], + unchanged: record => [record.p, record.n, record.q, record.q, record.c, '', ''] } - } - - return formatCsv(headers, rows); + ); }; /** @@ -238,77 +224,48 @@ const diffCollections = ( }; /** - * Create a light diff report between old and new collections. + * Create a diff report with annotated reasons between old and new collections. * - * @param diff - Diff report + * @param diff - Complete diff result */ const diffReport = (diff: ReturnType) => { - const { added, removed, modified } = diff; - const hasChanges = added.length > 0 || removed.length > 0 || modified.length > 0; - - console.log('\nšŸ“Š Collection Diff Report:'); - - if (!hasChanges) { - console.log(' ✨ No record additions, removals, or property modifications detected.'); - - return; - } - - if (added.length > 0) { - console.log(` āž• Added (${added.length}):`); - added.slice(0, 10).forEach(record => console.log(` + ${record.p} (Q: ${record.q})`)); - - if (added.length > 10) { - console.log(` ... and ${added.length - 10} more`); - } - } - - if (removed.length > 0) { - console.log(` āž– Removed (${removed.length}):`); - removed.slice(0, 15).forEach(({ record, reason, details }) => { - console.log(` - ${record.p} (Previous Q: ${record.q}) [Reason: ${reason}${details ? ` — ${details}` : ''}]`); - }); - - if (removed.length > 15) { - console.log(` ... and ${removed.length - 15} more`); - } - } - - if (modified.length > 0) { - console.log(` šŸ”„ Modified (${modified.length}):`); - modified.slice(0, 10).forEach(({ record, reasons }) => { - console.log(` ~ ${record.p} [${reasons.join(', ')}]`); - }); - - if (modified.length > 10) { - console.log(` ... and ${modified.length - 10} more`); - } - } + printDiffSummary(diff, { + title: 'PatternFly API Collection Diff Report', + formatAdded: record => `${record.p} (${record.n})`, + formatRemoved: ({ record, reason, details }) => `${record.p} (${record.n}) [Reason: ${reason}${details ? ` — ${details}` : ''}]`, + formatModified: ({ record, reasons }) => `${record.p} [${reasons.join(', ')}]` + }); }; /** * Run apiSpider directly and transform crawler entries into compressed embedded JSON. * * @param [options] - Optional configuration options. + * @param [options.filename=DEFAULT_API_FILENAME] - Target collection filename. + * @param [options.reportFilename=DEFAULT_API_REPORT_FILENAME] - Target CSV report filename. + * @param [options.outputPath] - Fully resolved path to write the JSON collection. + * @param [options.csvOutputPath] - Fully resolved path to write the CSV report. * @param [options.isPrettyPrint=true] - Whether to pretty-print the JSON output. * @param [options.filterLowQualityRecords=false] - Whether to filter low-quality records based on the collection's criteria. * @param [options.outputCsv=true] - Whether to generate and save a full CSV diff report. - * @param [options.csvOutputPath] - Custom path to write CSV report. */ const run = async ( { + filename = DEFAULT_API_FILENAME, + reportFilename = DEFAULT_API_REPORT_FILENAME, + outputPath = getSrcPath(filename), + csvOutputPath = getDefaultReportPath(reportFilename), isPrettyPrint = true, filterLowQualityRecords = false, - outputCsv = true, - csvOutputPath - }: { isPrettyPrint?: boolean; filterLowQualityRecords?: boolean; outputCsv?: boolean; csvOutputPath?: string; } = {} + outputCsv = true + }: UpdateApiOptions = {} ) => { // 1. Enable stderr logging so all diagnostics_channel logs (debug, info, warn, error) are printed const unsubscribeLogger = createLogger({ - channelName: getSessionOptions().channelName, + ...getLoggerOptions(), stderr: true, level: 'debug' - } as LoggingSession); + }); console.log('šŸš€ Generating PatternFly API embedded collection...'); const keepAlive = setTimeout(() => {}, 86_400_000); @@ -361,8 +318,6 @@ const run = async ( records }; - const outputPath = resolve(fileURLToPath(new URL('../src/collection.patternFlyApi.json', import.meta.url))); - const jsonContent = isPrettyPrint ? JSON.stringify(payload, null, 2) : JSON.stringify(payload); let oldRecords: ApiEmbedded[] = []; try { @@ -374,31 +329,23 @@ const run = async ( // File might not exist yet on initial run } - await writeFile(outputPath, jsonContent + '\n', 'utf-8'); - - const durationSec = ((Date.now() - startTime) / 1000).toFixed(1); - const sizeKb = (Buffer.byteLength(jsonContent, 'utf-8') / 1024).toFixed(1); - - console.log(`āœ… Updated src/collection.patternFlyApi.json:`); - console.log(` - Total Crawled: ${entries.length} endpoints`); - console.log(` - Admitted Records: ${records.length}`); - console.log(` - File Size: ${sizeKb} KB`); - console.log(` - Time Elapsed: ${durationSec}s`); + await writeJsonCollection(outputPath, payload, { + isPrettyPrint, + startTime, + stats: [ + { label: 'Total Crawled', value: `${entries.length} endpoints` }, + { label: 'Admitted Records', value: records.length } + ] + }); const diff = diffCollections(oldRecords, records, crawledMap); diffReport(diff); if (outputCsv) { - const targetCsvPath = csvOutputPath || - process.env.CSV_REPORT_PATH || - resolve(fileURLToPath(new URL('../reports/collection.patternFlyApi.report.csv', import.meta.url))); - - await mkdir(dirname(targetCsvPath), { recursive: true }); const csvContent = generateReportCsv({ diff, oldRecords, newRecords: records, crawledMap }); - await writeFile(targetCsvPath, csvContent, 'utf-8'); - console.log(`šŸ“„ Exported full CSV report: ${targetCsvPath}`); + await saveCsvReport(csvOutputPath, csvContent); } } finally { clearTimeout(keepAlive); @@ -407,23 +354,20 @@ const run = async ( }; /** - * Configurable options for maintainers. - * Only execute when explicitly requested via UPDATE_COLLECTIONS=true + * Direct execution when invoked via UPDATE_COLLECTIONS=true */ -if (process.env.UPDATE_COLLECTIONS === 'true') { - run({ isPrettyPrint: true, filterLowQualityRecords: true }).catch(error => { - console.error('āŒ Failed to update API collection:', error); - process.exit(1); - }); -} +runUpdateTask('API collection', run); export { + DEFAULT_API_FILENAME, + DEFAULT_API_REPORT_FILENAME, diffCollections, - escapeCsvField, - formatCsv, + diffReport, generateReportCsv, run, + type GenerateCsvReportOptions, type ModifiedRecordReport, type RemovalReason, - type RemovedRecordReport + type RemovedRecordReport, + type UpdateApiOptions }; diff --git a/tooling/collections/update.patternFlyDocs.ts b/tooling/collections/update.patternFlyDocs.ts new file mode 100644 index 00000000..14bcbb04 --- /dev/null +++ b/tooling/collections/update.patternFlyDocs.ts @@ -0,0 +1,178 @@ +import { existsSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import { type PatternFlyMcpDocsCatalog } from '../../src/docs.embedded'; +import { + getDefaultReportPath, + saveCsvReport +} from './csv'; +import { + extractTrackedReposFromCatalog, + fetchLatestRepoHashes, + getSrcPath, + runUpdateTask, + writeJsonCollection +} from './helpers'; +import { printDiffSummary } from './summary'; +import { + diffDocsManifests, + findApiRedundantDocs, + generateReportCsv, + recalculateManifestMetadata, + type ApiCollectionRecordRef, + type DocsDiffResult +} from './update.patternFlyDocsHelpers'; + +/** + * Options for running the documentation manifest collection update. + */ +interface UpdateDocsOptions { + docsFilename?: string | undefined; + apiFilename?: string | undefined; + reportFilename?: string | undefined; + docsPath?: string | undefined; + apiPath?: string | undefined; + csvOutputPath?: string | undefined; + pruneApiOverlap?: boolean | undefined; + verifyReachability?: boolean | undefined; + updateHashes?: boolean | undefined; + isPrettyPrint?: boolean | undefined; + outputCsv?: boolean | undefined; +} + +/** + * Configurable default filename for the documentation collection. + */ +const DEFAULT_DOCS_FILENAME = 'docs.json'; + +/** + * Configurable default filename for the API collection. + */ +const DEFAULT_API_FILENAME = 'collection.patternFlyApi.json'; + +/** + * Configurable default filename for the documentation report. + */ +const DEFAULT_DOCS_REPORT_FILENAME = 'collection.patternFlyDocs.report.csv'; + +/** + * Log a structured diff summary of the documentation changes to the console. + * + * @param diff - Complete diff result + */ +const logDiffReport = (diff: DocsDiffResult) => { + printDiffSummary(diff, { + title: 'PatternFly Docs Collection Diff Report', + formatAdded: item => `[${item.category}] ${item.record.displayName} (${item.record.pathSlug})`, + formatRemoved: item => `[${item.category}] ${item.record.displayName} (${item.record.pathSlug}) [Reason: ${item.reason}${item.details ? ` — ${item.details}` : ''}]`, + formatModified: item => `[${item.category}] ${item.record.displayName} [${item.reasons.join(', ')}]` + }); +}; + +/** + * Update the PatternFly Docs manifest collection, deduplicating against the API seed and syncing repository SHAs. + * + * @param [options] - Optional execution options. + * @param [options.docsFilename=DEFAULT_DOCS_FILENAME] - Target documentation filename. + * @param [options.apiFilename=DEFAULT_API_FILENAME] - Target API filename for deduplication. + * @param [options.reportFilename=DEFAULT_DOCS_REPORT_FILENAME] - Target CSV report filename. + * @param [options.docsPath] - Fully resolved path to write the documentation collection. + * @param [options.apiPath] - Fully resolved path to read the API collection. + * @param [options.csvOutputPath] - Fully resolved path to write the CSV report. + * @param [options.pruneApiOverlap=true] - Whether to prune duplicate docs that exist in API collection. + * @param [options.verifyReachability=false] - Whether to probe URLs for reachability. + * @param [options.updateHashes=true] - Whether to sync repository commit SHAs. + * @param [options.isPrettyPrint=true] - Whether to pretty-print the JSON output. + * @param [options.outputCsv=true] - Whether to generate and save CSV report. + * @returns Promise resolving to the diff result + */ +const run = async ({ + docsFilename = DEFAULT_DOCS_FILENAME, + apiFilename = DEFAULT_API_FILENAME, + reportFilename = DEFAULT_DOCS_REPORT_FILENAME, + docsPath = getSrcPath(docsFilename), + apiPath = getSrcPath(apiFilename), + csvOutputPath = getDefaultReportPath(reportFilename), + pruneApiOverlap = true, + verifyReachability = false, + updateHashes = true, + isPrettyPrint = true, + outputCsv = true +}: UpdateDocsOptions = {}): Promise => { + console.log('šŸš€ Updating PatternFly Docs manifest collection...'); + const startTime = Date.now(); + + // 1. Read existing documentation catalog + const rawDocs = await readFile(docsPath, 'utf-8'); + const oldCatalog: PatternFlyMcpDocsCatalog = JSON.parse(rawDocs); + + // 2. Read API collection for deduplication if enabled and present + let apiRecords: ApiCollectionRecordRef[] = []; + + if (pruneApiOverlap && existsSync(apiPath)) { + try { + const rawApi = await readFile(apiPath, 'utf-8'); + const apiCatalog = JSON.parse(rawApi); + + apiRecords = apiCatalog.records || []; + } catch { + console.warn(`āš ļø Could not parse API collection at ${apiPath}. Skipping API deduplication.`); + } + } + + // 3. Identify redundant records superseded by API collection + const redundantRecords = pruneApiOverlap ? findApiRedundantDocs(oldCatalog, apiRecords) : []; + + // 4. Fetch latest commit SHAs for tracked upstream repositories if enabled + let latestHashes = new Map(); + + if (updateHashes) { + const trackedRepos = extractTrackedReposFromCatalog(oldCatalog); + + latestHashes = await fetchLatestRepoHashes(trackedRepos); + } + + // 5. Build updated catalog and recalculate manifest metadata + const updatedCatalog = recalculateManifestMetadata(oldCatalog, { + redundantRecords, + latestHashes: updateHashes ? latestHashes : undefined, + verifyReachability + }); + + // 6. Write updated documentation manifest + await writeJsonCollection(docsPath, updatedCatalog, { + isPrettyPrint, + startTime, + stats: [ + { label: 'Total Categories', value: updatedCatalog.meta.totalEntries }, + { label: 'Total Documents', value: updatedCatalog.meta.totalDocs } + ] + }); + + // 7. Calculate diff and print console summary + const diff = diffDocsManifests(oldCatalog, updatedCatalog, redundantRecords); + + logDiffReport(diff); + + // 8. Generate and save CSV report if requested + if (outputCsv) { + const csvContent = generateReportCsv({ diff }); + + await saveCsvReport(csvOutputPath, csvContent); + } + + return diff; +}; + +/** + * Direct execution when invoked via UPDATE_COLLECTIONS=true + */ +runUpdateTask('Docs collection', run); + +export { + DEFAULT_API_FILENAME, + DEFAULT_DOCS_FILENAME, + DEFAULT_DOCS_REPORT_FILENAME, + logDiffReport, + run, + type UpdateDocsOptions +}; diff --git a/tooling/collections/update.patternFlyDocsHelpers.ts b/tooling/collections/update.patternFlyDocsHelpers.ts new file mode 100644 index 00000000..d169e6d6 --- /dev/null +++ b/tooling/collections/update.patternFlyDocsHelpers.ts @@ -0,0 +1,464 @@ +import { + type PatternFlyMcpDocsCatalog, + type PatternFlyMcpDocsCatalogDoc, + type PatternFlyMcpDocsCatalogEntry +} from '../../src/docs.embedded'; +import { extractCommitHash, extractRepoInfo } from './helpers'; +import { generateDiffCsv } from './csv'; + +/** + * Report entry for an added document in the manifest. + */ +interface DocsAddedRecordReport { + category: string; + record: PatternFlyMcpDocsCatalogDoc; + reason?: string | undefined; + details?: string | undefined; +} + +/** + * Report entry for a removed document in the manifest. + */ +interface DocsRemovedRecordReport { + category: string; + record: PatternFlyMcpDocsCatalogDoc; + reason: string; + details?: string | undefined; +} + +/** + * Report entry for a modified document in the manifest. + */ +interface DocsModifiedRecordReport { + category: string; + record: PatternFlyMcpDocsCatalogDoc; + previousRecord?: PatternFlyMcpDocsCatalogDoc | undefined; + reasons: string[]; + previousHash?: string | undefined; + newHash?: string | undefined; +} + +/** + * Report entry for an unchanged document in the manifest. + */ +interface DocsUnchangedRecordReport { + category: string; + record: PatternFlyMcpDocsCatalogDoc; +} + +/** + * Complete diff result between two documentation manifests. + */ +interface DocsDiffResult { + added: DocsAddedRecordReport[]; + removed: DocsRemovedRecordReport[]; + modified: DocsModifiedRecordReport[]; + unchanged: DocsUnchangedRecordReport[]; +} + +/** + * Options for generating Docs collection CSV diff report. + */ +interface GenerateCsvReportOptions { + diff: DocsDiffResult; +} + +/** + * Minimal shape of an API embedded collection record for cross-referencing. + */ +interface ApiCollectionRecordRef { + p: string; + n?: string | undefined; + q?: number | undefined; +} + +/** + * Options for recalculating manifest metadata and transforming entries. + */ +interface RecalculateOptions { + redundantRecords?: DocsRemovedRecordReport[] | undefined; + latestHashes?: Map | undefined; + verifyReachability?: boolean | undefined; +} + +/** + * Concise mapping of non-standard slugs to their corresponding API endpoint paths. + */ +const AI_GUIDELINE_ALIASES: Readonly> = Object.freeze({ + 'development-rules': 'overview', + guidelines: 'overview', + 'table-rules': 'table', + 'layout-components': 'layout', + troubleshooting: 'common-issues' +}); + +/** + * Pinned historical commit SHAs preserved for legacy or static references. + */ +const PINNED_HISTORICAL_REFS = new Set([ + 'ec02b437ec72b6e4cc4e28524516288f4acf9fdf', // Legacy design guidelines + 'ce032cd16ddb90c540cb4f18c6830e190cd9e3e9', // Tooling reference + '402b3b0e7ed73cb2aa21531e0eab4216c2211212', // Elements reference + 'e8cca17430a8ccb062ed1878073165417a081b34' // AIHelpers pinned baseline +]); + +/** + * Resolve the matching API endpoint for an upstream AI helper document. + * + * @param doc - Document record to evaluate + * @param highQualityApi - List of API collection records meeting the quality threshold + * @returns Matched API endpoint path or undefined if not superseded + */ +const resolveApiEndpointForAiDoc = ( + doc: PatternFlyMcpDocsCatalogDoc, + highQualityApi: ApiCollectionRecordRef[] +): string | undefined => { + const slug = doc.pathSlug || ''; + + // 1. Root ai-helpers links superseded by the marketplace endpoint + if (/^ai-helpers-(readme|contributing|contributing-skills)$/.test(slug)) { + return 'v6/AI/ai-assisted-development_marketplace/text'; + } + + // 2. Component and development guidelines in ai-helpers + if (doc.path && doc.path.includes('/docs/')) { + const targetSlug = AI_GUIDELINE_ALIASES[slug] || slug; + const candidate = `v6/AI/development-guidelines_${targetSlug}/text`; + const exactMatch = highQualityApi.find(apiRecord => apiRecord.p === candidate); + + if (exactMatch) { + return exactMatch.p; + } + + // Dynamic token fallback against indexed API endpoints + const token = targetSlug.toLowerCase().replace(/[^a-z0-9]/g, ''); + const fuzzyMatch = highQualityApi.find( + apiRecord => + apiRecord.p.startsWith('v6/AI/development-guidelines') && + apiRecord.p.toLowerCase().replace(/[^a-z0-9]/g, '').includes(token) + ); + + return fuzzyMatch ? fuzzyMatch.p : candidate; + } + + return undefined; +}; + +/** + * Identify documentation entries in `docs.json` that are redundant with and superseded by the PatternFly API collection. + * + * @param docsCatalog - The documentation catalog to inspect + * @param [apiRecords=[]] - Records from the API collection seed + * @returns List of redundant records with removal reasons and matched API details + */ +const findApiRedundantDocs = ( + docsCatalog: PatternFlyMcpDocsCatalog, + apiRecords: ApiCollectionRecordRef[] = [] +): DocsRemovedRecordReport[] => { + const redundant: DocsRemovedRecordReport[] = []; + const highQualityApi = (apiRecords || []).filter(apiRecord => (apiRecord.q ?? 1) >= 0.95); + + for (const [category, entries] of Object.entries(docsCatalog.docs || {})) { + for (const doc of entries) { + // 1. Root uxd-ai-helpers guides must NEVER be pruned (negative control boundary) + if (doc.pathSlug && doc.pathSlug.startsWith('uxd-ai-helpers-')) { + continue; + } + + // 2. Ecosystem / tooling repos must NEVER be pruned + if ( + doc.path && + (doc.path.includes('/patternfly-cli/') || + doc.path.includes('/patternfly-elements/') || + doc.path.includes('/patternfly-mcp/') || + doc.path.includes('/pf-codemods/')) + ) { + continue; + } + + const isAiHelperRepo = + doc.path && + (doc.path.includes('/ai-helpers/') || doc.path.includes('/uxd-ai-helpers/')); + + if (isAiHelperRepo) { + const details = resolveApiEndpointForAiDoc(doc, highQualityApi); + + if (details) { + redundant.push({ + category, + record: doc, + reason: 'superseded by API collection', + details + }); + } + } + } + } + + return redundant; +}; + +/** + * Calculate the diff between an old documentation catalog and an updated catalog. + * + * @param oldCatalog - Original catalog + * @param newCatalog - Updated catalog + * @param [redundantReports=[]] - Explicitly identified removals + * @returns Structured DocsDiffResult + */ +const diffDocsManifests = ( + oldCatalog: PatternFlyMcpDocsCatalog, + newCatalog: PatternFlyMcpDocsCatalog, + redundantReports: DocsRemovedRecordReport[] = [] +): DocsDiffResult => { + const added: DocsAddedRecordReport[] = []; + const removed: DocsRemovedRecordReport[] = [...redundantReports]; + const modified: DocsModifiedRecordReport[] = []; + const unchanged: DocsUnchangedRecordReport[] = []; + + const oldRecordsMap = new Map(); + + for (const [cat, docs] of Object.entries(oldCatalog.docs || {})) { + for (const doc of docs) { + oldRecordsMap.set(`${cat}::${doc.pathSlug}::${doc.displayName}`, { category: cat, doc }); + } + } + + const newRecordsMap = new Map(); + + for (const [cat, docs] of Object.entries(newCatalog.docs || {})) { + for (const doc of docs) { + newRecordsMap.set(`${cat}::${doc.pathSlug}::${doc.displayName}`, { category: cat, doc }); + } + } + + const removedKeys = new Set( + redundantReports.map( + report => `${report.category}::${report.record.pathSlug}::${report.record.displayName}` + ) + ); + + // Check for any additional old records missing in new catalog + for (const [key, { category, doc }] of oldRecordsMap.entries()) { + if (!newRecordsMap.has(key) && !removedKeys.has(key)) { + removed.push({ + category, + record: doc, + reason: 'removed from manifest', + details: 'Record excluded during manifest update' + }); + removedKeys.add(key); + } + } + + // Check for added, modified, or unchanged in new catalog + for (const [key, { category, doc }] of newRecordsMap.entries()) { + const oldEntry = oldRecordsMap.get(key); + + if (!oldEntry) { + added.push({ + category, + record: doc, + reason: 'new upstream document', + details: 'Discovered in manifest update' + }); + continue; + } + + const reasons: string[] = []; + const previousDoc = oldEntry.doc; + const oldHash = extractCommitHash(previousDoc.path || ''); + const newHash = extractCommitHash(doc.path || ''); + + if (previousDoc.path !== doc.path) { + if (oldHash !== newHash && oldHash && newHash) { + const shortOld = oldHash.slice(0, 7); + const shortNew = newHash.slice(0, 7); + + reasons.push(`hash update (${shortOld} -> ${shortNew})`); + } else { + reasons.push('path updated'); + } + } + + if (previousDoc.version !== doc.version) { + reasons.push(`version changed (${previousDoc.version} -> ${doc.version})`); + } + + if (reasons.length > 0) { + modified.push({ + category, + record: doc, + previousRecord: previousDoc, + reasons, + previousHash: oldHash || undefined, + newHash: newHash || undefined + }); + } else { + unchanged.push({ + category, + record: doc + }); + } + } + + return { added, removed, modified, unchanged }; +}; + +/** + * Generate a complete RFC 4180 CSV report for the documentation manifest diff. + * + * @param options - CSV report options or diff result + * @returns Formatted CSV content string + */ +const generateReportCsv = (options: DocsDiffResult | GenerateCsvReportOptions): string => { + const diff = 'diff' in options ? options.diff : options; + + return generateDiffCsv(diff, { + headers: [ + 'status', + 'category', + 'name', + 'pathSlug', + 'path', + 'previousHash', + 'newHash', + 'reason', + 'details' + ], + added: item => [ + item.category, + item.record.displayName, + item.record.pathSlug, + item.record.path, + '', + extractCommitHash(item.record.path) || '', + item.reason || 'new document', + item.details || '' + ], + removed: item => [ + item.category, + item.record.displayName, + item.record.pathSlug, + item.record.path, + extractCommitHash(item.record.path) || '', + '', + item.reason, + item.details || '' + ], + modified: item => [ + item.category, + item.record.displayName, + item.record.pathSlug, + item.record.path, + item.previousHash || '', + item.newHash || '', + item.reasons.join('; '), + item.previousRecord?.path || '' + ], + unchanged: item => { + const hash = extractCommitHash(item.record.path) || ''; + + return [ + item.category, + item.record.displayName, + item.record.pathSlug, + item.record.path, + hash, + hash, + '', + '' + ]; + } + }); +}; + +/** + * Recalculate catalog entries, prune redundant records, update hashes, and recompute manifest metadata. + * + * @param catalog - Source documentation catalog + * @param [options={}] - Options for recalculation + * @returns Updated PatternFlyMcpDocsCatalog + */ +const recalculateManifestMetadata = ( + catalog: PatternFlyMcpDocsCatalog, + options: RecalculateOptions = {} +): PatternFlyMcpDocsCatalog => { + const redundantSet = new Set( + (options.redundantRecords || []).map( + report => `${report.category}::${report.record.pathSlug}::${report.record.displayName}` + ) + ); + + const updatedDocs: PatternFlyMcpDocsCatalogEntry = {}; + + for (const [category, entries] of Object.entries(catalog.docs || {})) { + const filteredEntries: PatternFlyMcpDocsCatalogDoc[] = []; + + for (const doc of entries) { + const key = `${category}::${doc.pathSlug}::${doc.displayName}`; + + if (redundantSet.has(key)) { + continue; + } + + let updatedPath = doc.path; + + // Update commit hash if new hash is available and doc is not a pinned one-off / v5 + if (options.latestHashes && doc.path && typeof doc.path === 'string') { + const repoInfo = extractRepoInfo(doc.path); + + if (repoInfo && repoInfo.ref !== 'v5') { + const repoKey = `${repoInfo.owner}/${repoInfo.repo}`; + const newSha = options.latestHashes.get(repoKey) || options.latestHashes.get(repoInfo.repo); + + // Update if repo is recognized and not a pinned historical reference + if (newSha && !PINNED_HISTORICAL_REFS.has(repoInfo.ref)) { + updatedPath = `https://raw.githubusercontent.com/${repoInfo.owner}/${repoInfo.repo}/${newSha}/${repoInfo.filePath}`; + } + } + } + + filteredEntries.push({ + ...doc, + path: updatedPath + }); + } + + if (filteredEntries.length > 0) { + updatedDocs[category] = filteredEntries; + } + } + + const totalEntries = Object.keys(updatedDocs).length; + const totalDocs = Object.values(updatedDocs).reduce((acc, arr) => acc + arr.length, 0); + + return { + version: catalog.version || '1', + generated: new Date().toISOString(), + meta: { + totalEntries, + totalDocs, + source: catalog.meta?.source || 'patternfly-mcp' + }, + docs: updatedDocs + }; +}; + +export { + AI_GUIDELINE_ALIASES, + PINNED_HISTORICAL_REFS, + diffDocsManifests, + findApiRedundantDocs, + generateReportCsv, + recalculateManifestMetadata, + resolveApiEndpointForAiDoc, + type ApiCollectionRecordRef, + type GenerateCsvReportOptions, + type DocsAddedRecordReport, + type DocsDiffResult, + type DocsModifiedRecordReport, + type DocsRemovedRecordReport, + type DocsUnchangedRecordReport, + type RecalculateOptions +}; diff --git a/tsconfig.json b/tsconfig.json index 6a5320a5..95b6bcc9 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -21,7 +21,7 @@ "resolveJsonModule": true, "noEmit": true, "stripInternal": true, - "rootDirs": ["./src", "./scripts", "./tests/e2e", "./tests/audit"], + "rootDirs": ["./src", "./scripts", "./tooling", "./tests/e2e", "./tests/audit"], "paths": { "#apiCatalog": ["./src/collection.patternFlyApi.json"], "#docsCatalog": ["./src/docs.json"], @@ -34,6 +34,7 @@ "src/**/*", "tests/**/*", "scripts/**/*", + "tooling/**/*", "jest.setupTests.ts", "jest.config.ts", "types/**/*.d.ts",