From df9a941e2cf0753eae91fcffc3901609c67d19d4 Mon Sep 17 00:00:00 2001 From: CD Cabrera Date: Fri, 9 Oct 2026 14:31:47 -0400 Subject: [PATCH 1/5] feat(tooling): pf-4720 pf-docs collection * tooling/collections, add pf-docs collection script for maintainer updates --- docs/development.md | 21 +- guidelines/README.md | 2 +- package.json | 3 +- src/__tests__/docs.json.test.ts | 2 +- tests/audit/api.audit.test.ts | 8 +- tests/audit/docs.audit.test.ts | 10 +- tooling/collections/__tests__/helpers.test.ts | 422 ++++++++++++++ .../__tests__/update.patternFlyDocs.test.ts | 455 +++++++++++++++ .../update.patternFlyDocsHelpers.test.ts | 509 ++++++++++++++++ tooling/collections/helpers.ts | 282 +++++++++ tooling/collections/update.patternFlyDocs.ts | 179 ++++++ .../update.patternFlyDocsHelpers.ts | 545 ++++++++++++++++++ 12 files changed, 2424 insertions(+), 14 deletions(-) create mode 100644 tooling/collections/__tests__/update.patternFlyDocs.test.ts create mode 100644 tooling/collections/__tests__/update.patternFlyDocsHelpers.test.ts create mode 100644 tooling/collections/update.patternFlyDocs.ts create mode 100644 tooling/collections/update.patternFlyDocsHelpers.ts diff --git a/docs/development.md b/docs/development.md index f0b1f6e9..b61e704b 100644 --- a/docs/development.md +++ b/docs/development.md @@ -544,8 +544,9 @@ The server maintains multiple record collections that provide documentation and #### General collection reseeding When PatternFly releases new documentation or component features, reseed the embedded local collection data: -- `npm run build:collections`: Runs the complete crawler and manifest update pipeline and runs collection tests. +- `npm run build:collections`: Runs reseeding for all collections and collection tests. - `npm run build:collections-api`: Updates `src/collection.patternFlyApi.json` and outputs CSV diffs to `reports/`. +- `npm run build:collections-docs`: Updates `src/docs.json` with latest repository hashes and prunes API duplicates. #### Unique collections @@ -555,9 +556,20 @@ When PatternFly releases new documentation or component features, reseed the emb The original curated Markdown collection. It pins remote repository URLs to specific Git commit SHAs to guarantee consistent documentation and remove upstream breaking changes. -- **Update workflow**: Follow the [add-docs-links skill](../guidelines/skills/add-docs-links/SKILL.md) to add, update, or remove entries. +- **Update workflow**: Follow the [add-docs-links skill](../guidelines/skills/add-docs-links/SKILL.md) for manual additions/edits, or run the automated reseeding script: + ```bash + # Reseed the Docs collection + npm run build:collections-docs + + # Run collection test suites only + npm run test:collections + + # Run collection test suites in watch mode + npm run test:collections-dev + ``` +- **Automated synchronization**: The Docs reseeding pipeline (`tooling/collections/update.patternFlyDocs.ts`) automatically resolves the latest commit SHAs across referenced GitHub repositories, prunes redundant AI guideline entries superseded by high-quality API endpoints, recalculates `meta.totalEntries` and `meta.totalDocs`, and generates an audit diff report at `reports/YYYYMMDD-collection.patternFlyDocs.report.csv`. - **Validation**: Ensure all raw URLs are HTTPS, match the domain whitelist in `src/options.defaults.ts`, and return 2xx responses. -- **Testing**: Run `npm test` and update `baseHashes` and the Repository Breakdown table in `src/__tests__/docs.json.test.ts` whenever refs change. +- **Testing**: Run `npm run test:collections` or `npm test`, updating `baseHashes` and the Repository Breakdown table in `src/__tests__/docs.json.test.ts` when refs change. ##### 2. Pre-built API collection (`src/collection.patternFlyApi.json`) @@ -565,9 +577,6 @@ The server packages pre-seeded API endpoints to enable instant server startup wi - **Update commands**: ```bash - # Reseed the API collection and run collection test suites - npm run build:collections - # Reseed the API collection npm run build:collections-api diff --git a/guidelines/README.md b/guidelines/README.md index a4756fa1..83b13db2 100644 --- a/guidelines/README.md +++ b/guidelines/README.md @@ -37,7 +37,7 @@ Agents should use these phrases as signals to consult specific documentation and | **"review development guide"** | Review `docs/development.md` for CLI, API, and plugin authoring. | | **"create an example tool plugin"** | Review `guidelines/agent_coding.md`, `docs/development.md`, `docs/examples/*`, and `src/*` for context, coding standards, and existing example formats. | | **"add documentation links"** / **"add doc entries"** / **"register docs"** / **"update docs.json"** / **"contribute to docs.json"** | Follow `guidelines/skills/add-docs-links/SKILL.md`: docs.json format, duplicate check, raw URL confirmation, then run unit tests and update meta. | -| **"reseed collections"** / **"update collections"** / **"build collections"** | Follow `docs/development.md#updating-collections`: run `npm run build:collections` (or `build:collections-api`) to sync API records. | +| **"reseed collections"** / **"update collections"** / **"build collections"** / **"refresh docs manifest"** | Follow `docs/development.md#updating-collections`: run `npm run build:collections` (or `build:collections-api` / `build:collections-docs`) to sync API records and docs manifest. | | **"review zod"** / **"zod upgrade"** / **"zod integration review"** / **"zod update report"** | Follow `guidelines/skills/review-zod-integration/SKILL.md`: release notes vs codebase, tests, report at `reports/YYYYMMDD-HHMMSS-zod-{semver}-update-report.md` (UTC). | | **"troubleshoot server"** / **"debug server"** | Review `docs/usage.md#troubleshooting` and the PatternFly MCP server resource `patternfly://context` | diff --git a/package.json b/package.json index 9395e3b5..edf28e94 100644 --- a/package.json +++ b/package.json @@ -35,8 +35,9 @@ "build": "npm run build:clean; npm run test:types; pkgroll", "build:clean": "rm -rf dist", "build:watch": "npm run build -- --watch", - "build:collections": "npm run build:collections-api && jest --selectProjects tooling:collections", + "build:collections": "npm run build:collections-api && npm run build:collections-docs && jest --selectProjects tooling:collections", "build:collections-api": "UPDATE_COLLECTIONS=true tsx ./tooling/collections/update.patternFlyApi.ts", + "build:collections-docs": "UPDATE_COLLECTIONS=true tsx ./tooling/collections/update.patternFlyDocs.ts", "container:build": "bash ./scripts/container.build.sh", "container:start": "bash ./scripts/container.run.sh", "release": "changelog --non-cc --link-url https://github.com/patternfly/patternfly-mcp.git", diff --git a/src/__tests__/docs.json.test.ts b/src/__tests__/docs.json.test.ts index 20db5cf3..d4054f13 100644 --- a/src/__tests__/docs.json.test.ts +++ b/src/__tests__/docs.json.test.ts @@ -74,7 +74,7 @@ describe('docs.json', () => { * * | Repository | Hash / Branch | Count | Type | * |---|---|---|---| - * | uxd-ai-helpers | e8cca17430a8ccb062ed1878073165417a081b34 | 6 | **Primary** | + * | uxd-ai-helpers | e8cca17430a8ccb062ed1878073165417a081b34 | 6 | One-off | * | patternfly-cli | ce032cd16ddb90c540cb4f18c6830e190cd9e3e9 | 1 | One-off | * | patternfly-elements | 402b3b0e7ed73cb2aa21531e0eab4216c2211212 | 1 | One-off | * | patternfly-mcp | 4e1cd7c636f6ac17c851f54a920549e5377f3809 | 5 | One-off | diff --git a/tests/audit/api.audit.test.ts b/tests/audit/api.audit.test.ts index a353ab4e..6626a9df 100644 --- a/tests/audit/api.audit.test.ts +++ b/tests/audit/api.audit.test.ts @@ -4,10 +4,10 @@ import { expandApiEmbeddedCollection, type ApiEmbeddedCollection } from '../../s import { checkUrl } from './utils/checkUrl'; describe('PatternFly API Link Audit', () => { - const catalogPath = resolve(process.cwd(), 'src/collection.patternFlyApi.json'); - const raw = readFileSync(catalogPath, 'utf-8'); - const catalog: ApiEmbeddedCollection = JSON.parse(raw); - const expanded = expandApiEmbeddedCollection(catalog); + const collectionPath = resolve(process.cwd(), 'src/collection.patternFlyApi.json'); + const raw = readFileSync(collectionPath, 'utf-8'); + const collection: ApiEmbeddedCollection = JSON.parse(raw); + const expanded = expandApiEmbeddedCollection(collection); // Sample a subset across records const maxSample = Number(process.env.API_AUDIT_MAX_TOTAL ?? 20); diff --git a/tests/audit/docs.audit.test.ts b/tests/audit/docs.audit.test.ts index e410dd68..2c7beb26 100644 --- a/tests/audit/docs.audit.test.ts +++ b/tests/audit/docs.audit.test.ts @@ -1,11 +1,19 @@ import { randomInt } from 'node:crypto'; +import { existsSync, readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; import { jest } from '@jest/globals'; -import docs from '../../src/docs.json'; +import { type PatternFlyMcpDocsCatalog } from '../../src/docs.embedded'; import { checkUrl } from './utils/checkUrl'; describe('Documentation Link Audit', () => { let auditSet: string[] = []; + const collectionPath = resolve(process.cwd(), 'src/docs.json'); + + const docs: PatternFlyMcpDocsCatalog = existsSync(collectionPath) + ? JSON.parse(readFileSync(collectionPath, 'utf-8')) + : { meta: { totalEntries: 0, totalDocs: 0, source: '' }, docs: {} }; + /** * Extracts and returns a prefix based on the provided URL. * - If the hostname is not "raw.githubusercontent.com", the hostname is returned. diff --git a/tooling/collections/__tests__/helpers.test.ts b/tooling/collections/__tests__/helpers.test.ts index 98691bbb..eb4d6b03 100644 --- a/tooling/collections/__tests__/helpers.test.ts +++ b/tooling/collections/__tests__/helpers.test.ts @@ -1,8 +1,16 @@ +import * as childProcess from 'node:child_process'; import { writeFile } from 'node:fs/promises'; import { + extractCommitHash, + extractRepoInfo, + extractTrackedReposFromCollection, + fetchCommitViaGitLsRemote, + fetchRepoCommit, + fetchLatestRepoHashes, getReportsPath, getSrcPath, runUpdateTask, + verifyUrlReachability, writeJsonCollection } from '../helpers'; @@ -11,8 +19,422 @@ jest.mock('node:fs/promises', () => ({ writeFile: jest.fn() })); +jest.mock('node:child_process', () => ({ + ...jest.requireActual('node:child_process'), + execSync: jest.fn() +})); + const mockWriteFile = writeFile as jest.MockedFunction; +afterEach(() => { + extractRepoInfo.memo.clear(); + fetchRepoCommit.memo.clear(); + verifyUrlReachability.memo.clear(); +}); + +describe('extractTrackedReposFromCollection', () => { + it('should extract and deduplicate repositories across multiple categories', () => { + const mockCollection = { + version: '1', + generated: '2026-01-01', + meta: { totalEntries: 2, totalDocs: 3, source: 'test' }, + docs: { + components: [ + { displayName: 'Button', pathSlug: 'button', path: 'https://raw.githubusercontent.com/patternfly/patternfly-react/sha1/pkg/button.md' }, + { displayName: 'Card', pathSlug: 'card', path: 'https://raw.githubusercontent.com/patternfly/patternfly-react/sha2/pkg/card.md' } + ], + guidelines: [ + { displayName: 'AI Guidelines', pathSlug: 'ai', path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/sha3/docs/ai.md' }, + { displayName: 'Invalid Path', pathSlug: 'invalid', path: 'https://example.com/invalid.md' } + ] + } + }; + + const result = extractTrackedReposFromCollection(mockCollection as any); + + expect(result).toHaveLength(2); + expect(result).toEqual([ + { owner: 'patternfly', repo: 'patternfly-react', branch: undefined }, + { owner: 'rh-uxd', repo: 'ai-helpers', branch: undefined } + ]); + }); + + it('should return an empty array when collection contains no valid docs', () => { + expect(extractTrackedReposFromCollection({} as any)).toEqual([]); + expect(extractTrackedReposFromCollection({ docs: {} } as any)).toEqual([]); + }); + + it('should apply branch override when supplied', () => { + const collection = { docs: { c: [{ path: 'https://raw.githubusercontent.com/org/repo/sha/file.md' }] } }; + const result = extractTrackedReposFromCollection(collection as any, 'develop'); + + expect(result[0]?.branch).toBe('develop'); + }); +}); + +describe('extractCommitHash', () => { + it.each([ + { + description: 'full 40-character SHA raw GitHub URL', + url: 'https://raw.githubusercontent.com/patternfly/patternfly-org/540bb0d31cb18670dd02857f80aa8b444fed9be9/packages/documentation-site/patternfly-docs/content/AI/ai.md', + expected: '540bb0d31cb18670dd02857f80aa8b444fed9be9' + }, + { + description: 'branch ref raw GitHub URL', + url: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/docs/components/data-display/table.md', + expected: 'main' + }, + { + description: 'non-raw GitHub URL', + url: 'https://github.com/patternfly/patternfly-org/blob/main/README.md', + expected: undefined + }, + { + description: 'non-GitHub URL', + url: 'https://example.com/invalid', + expected: undefined + }, + { + description: 'empty string URL', + url: '', + expected: undefined + }, + { + description: 'null input', + url: null as unknown as string, + expected: undefined + }, + { + description: 'undefined input', + url: undefined as unknown as string, + expected: undefined + } + ])('should extract commit hash or return undefined, $description', ({ url, expected }) => { + expect(extractCommitHash(url)).toBe(expected); + }); +}); + +describe('extractRepoInfo', () => { + it.each([ + { + description: 'valid raw GitHub URL with subpath', + url: 'https://raw.githubusercontent.com/patternfly/patternfly-org/540bb0d31359c381c8152331575ca2481e3fe1ff/packages/v4/src/content/components/button.md', + expected: { + owner: 'patternfly', + repo: 'patternfly-org', + ref: '540bb0d31359c381c8152331575ca2481e3fe1ff', + filePath: 'packages/v4/src/content/components/button.md' + } + }, + { + description: 'plain invalid URL string', + url: 'invalid-url', + expected: undefined + }, + { + description: 'incomplete GitHub path structure', + url: 'https://raw.githubusercontent.com/incomplete', + expected: undefined + }, + { + description: 'empty string input', + url: '', + expected: undefined + }, + { + description: 'null input', + url: null as unknown as string, + expected: undefined + }, + { + description: 'undefined input', + url: undefined as unknown as string, + expected: undefined + } + ])('should extract repository metadata or return undefined, $description', ({ url, expected }) => { + expect(extractRepoInfo(url)).toEqual(expected); + }); + + it('should have a memo property', () => { + expect(extractRepoInfo.memo).toBeDefined(); + }); +}); + +describe('verifyUrlReachability', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should return true when HEAD request succeeds', async () => { + jest.spyOn(global, 'fetch').mockResolvedValue({ ok: true, status: 200 } as Response); + + const reachable = await verifyUrlReachability('https://raw.githubusercontent.com/test/file.md'); + + expect(reachable).toBe(true); + expect(global.fetch).toHaveBeenCalledWith( + 'https://raw.githubusercontent.com/test/file.md', + expect.objectContaining({ method: 'HEAD' }) + ); + }); + + it('should fall back to GET when HEAD returns 405 or 403', async () => { + jest + .spyOn(global, 'fetch') + .mockResolvedValueOnce({ ok: false, status: 405 } as Response) + .mockResolvedValueOnce({ ok: true, status: 200 } as Response); + + const reachable = await verifyUrlReachability('https://raw.githubusercontent.com/test/file.md'); + + expect(reachable).toBe(true); + expect(global.fetch).toHaveBeenCalledTimes(2); + expect(global.fetch).toHaveBeenLastCalledWith( + 'https://raw.githubusercontent.com/test/file.md', + expect.objectContaining({ method: 'GET' }) + ); + }); + + it('should return false when fetch throws network error or times out', async () => { + jest.spyOn(global, 'fetch').mockRejectedValue(new Error('Network error')); + + const reachable = await verifyUrlReachability('https://raw.githubusercontent.com/test/file.md'); + + expect(reachable).toBe(false); + }); + + it('should have a memo property that caches responses', async () => { + expect(verifyUrlReachability.memo).toBeDefined(); + + const fetchSpy = jest.spyOn(global, 'fetch').mockResolvedValue({ + ok: true, + status: 200 + } as Response); + + const first = await verifyUrlReachability.memo('https://raw.githubusercontent.com/patternfly/test.md'); + const second = await verifyUrlReachability.memo('https://raw.githubusercontent.com/patternfly/test.md'); + + expect(first).toBe(true); + expect(second).toBe(true); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); +}); + +describe('fetchCommitViaGitLsRemote', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should return commit SHA when git ls-remote output contains SHA with HEAD', () => { + jest.spyOn(childProcess, 'execSync').mockReturnValue( + 'ref: refs/heads/main\tHEAD\n8f4a382e783457a4128509789234857234895723\tHEAD\n' + ); + + const sha = fetchCommitViaGitLsRemote('patternfly', 'patternfly-react'); + + expect(sha).toBe('8f4a382e783457a4128509789234857234895723'); + }); + + it('should fallback to 40-character SHA if symref line is missing', () => { + jest.spyOn(childProcess, 'execSync').mockReturnValue( + '8f4a382e783457a4128509789234857234895723\trefs/heads/main\n' + ); + + const sha = fetchCommitViaGitLsRemote('patternfly', 'patternfly-react'); + + expect(sha).toBe('8f4a382e783457a4128509789234857234895723'); + }); + + it('should return undefined if command fails or throws', () => { + jest.spyOn(childProcess, 'execSync').mockImplementation(() => { + throw new Error('Command failed'); + }); + + const sha = fetchCommitViaGitLsRemote('patternfly', 'patternfly-react'); + + expect(sha).toBeUndefined(); + }); +}); + +describe('fetchRepoCommit', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('should resolve commit SHA for a specific branch from GitHub API', async () => { + jest.spyOn(global, 'fetch').mockResolvedValue({ + ok: true, + json: async () => ({ sha: 'mock-branch-sha' }) + } as Response); + + const sha = await fetchRepoCommit('patternfly', 'patternfly-react', 'main'); + + expect(sha).toBe('mock-branch-sha'); + expect(global.fetch).toHaveBeenCalledWith( + 'https://api.github.com/repos/patternfly/patternfly-react/commits/main', + expect.any(Object) + ); + }); + + it('should resolve commit SHA for default branch when branch is not specified', async () => { + jest.spyOn(global, 'fetch').mockResolvedValue({ + ok: true, + json: async () => [{ sha: 'mock-default-sha' }] + } as Response); + + const sha = await fetchRepoCommit('patternfly', 'patternfly-react'); + + expect(sha).toBe('mock-default-sha'); + expect(global.fetch).toHaveBeenCalledWith( + 'https://api.github.com/repos/patternfly/patternfly-react/commits', + expect.any(Object) + ); + }); + + it('should fallback to git ls-remote when GitHub API fails', async () => { + jest.spyOn(global, 'fetch').mockRejectedValue(new Error('Network error')); + jest.spyOn(childProcess, 'execSync').mockReturnValue( + '0123456789abcdef0123456789abcdef01234567\tHEAD\n' + ); + + const sha = await fetchRepoCommit('patternfly', 'patternfly-org'); + + expect(sha).toBe('0123456789abcdef0123456789abcdef01234567'); + }); + + it('should return undefined when API fails and git ls-remote fails', async () => { + jest.spyOn(global, 'fetch').mockRejectedValue(new Error('Network error')); + jest.spyOn(childProcess, 'execSync').mockImplementation(() => { + throw new Error('git error'); + }); + + const sha = await fetchRepoCommit('patternfly', 'patternfly-org'); + + expect(sha).toBeUndefined(); + }); + + it('should have a memo property that caches responses', async () => { + expect(fetchRepoCommit.memo).toBeDefined(); + + const fetchSpy = jest.spyOn(global, 'fetch').mockResolvedValue({ + ok: true, + json: async () => ({ sha: 'mock-memo-sha' }) + } as Response); + + const first = await fetchRepoCommit.memo('patternfly', 'patternfly-react', 'main'); + const second = await fetchRepoCommit.memo('patternfly', 'patternfly-react', 'main'); + + expect(first).toBe('mock-memo-sha'); + expect(second).toBe('mock-memo-sha'); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); +}); + +describe('fetchLatestRepoHashes', () => { + const originalEnv = process.env; + + beforeEach(() => { + process.env = { ...originalEnv }; + }); + + afterEach(() => { + process.env = originalEnv; + jest.restoreAllMocks(); + }); + + it('should fetch commit SHAs from GitHub API and index by owner/repo and repo name', async () => { + jest.spyOn(global, 'fetch').mockImplementation(async (input: RequestInfo | URL) => { + const url = typeof input === 'string' ? input : input.toString(); + + if (url.includes('patternfly-react')) { + return { + ok: true, + json: async () => ({ sha: 'mock-react-sha-123' }) + } as Response; + } + + return { ok: false, status: 404 } as Response; + }); + + const repos = [ + { owner: 'patternfly', repo: 'patternfly-react', branch: 'main' }, + { owner: 'patternfly', repo: 'nonexistent', branch: 'main' } + ]; + + const hashes = await fetchLatestRepoHashes(repos); + + expect(hashes.get('patternfly/patternfly-react')).toBe('mock-react-sha-123'); + expect(hashes.get('patternfly-react')).toBe('mock-react-sha-123'); + expect(hashes.has('patternfly/nonexistent')).toBe(false); + }); + + it('should query commits list when branch is not specified and handle array response', async () => { + let capturedUrl = ''; + + jest.spyOn(global, 'fetch').mockImplementation(async (input: RequestInfo | URL) => { + capturedUrl = typeof input === 'string' ? input : input.toString(); + + return { + ok: true, + json: async () => [{ sha: 'mock-default-sha-456' }] + } as Response; + }); + + const hashes = await fetchLatestRepoHashes([ + { owner: 'patternfly', repo: 'patternfly-elements' } + ]); + + expect(capturedUrl).toBe('https://api.github.com/repos/patternfly/patternfly-elements/commits'); + expect(hashes.get('patternfly/patternfly-elements')).toBe('mock-default-sha-456'); + expect(hashes.get('patternfly-elements')).toBe('mock-default-sha-456'); + }); + + it('should send standard unauthenticated headers with User-Agent and Accept without Authorization', async () => { + let capturedHeaders: Record = {}; + + jest.spyOn(global, 'fetch').mockImplementation(async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedHeaders = (init?.headers as Record) || {}; + + return { + ok: true, + json: async () => ({ sha: 'mock-public-sha' }) + } as Response; + }); + + await fetchLatestRepoHashes([ + { owner: 'patternfly', repo: 'patternfly-mcp', branch: 'main' } + ]); + + expect(capturedHeaders['User-Agent']).toBe('patternfly-mcp'); + expect(capturedHeaders.Accept).toBe('application/vnd.github.v3+json'); + expect(capturedHeaders.Authorization).toBeUndefined(); + }); + + it('should fall back to git ls-remote when GitHub API fails', async () => { + jest.spyOn(global, 'fetch').mockRejectedValue(new Error('Rate limit exceeded')); + jest.spyOn(childProcess, 'execSync').mockReturnValue( + '0123456789abcdef0123456789abcdef01234567\tHEAD\n' + ); + + const hashes = await fetchLatestRepoHashes([ + { owner: 'patternfly', repo: 'patternfly-org', branch: 'main' } + ]); + + expect(hashes.get('patternfly/patternfly-org')).toBe('0123456789abcdef0123456789abcdef01234567'); + }); + + it('should handle fetch failures gracefully without throwing', async () => { + jest.spyOn(global, 'fetch').mockRejectedValue(new Error('Rate limit exceeded')); + jest.spyOn(childProcess, 'execSync').mockImplementation(() => { + throw new Error('git command failed'); + }); + + const hashes = await fetchLatestRepoHashes([ + { owner: 'patternfly', repo: 'patternfly-org', branch: 'main' } + ]); + + expect(hashes.size).toBe(0); + }); +}); + describe('writeJsonCollection', () => { let logSpy: ReturnType; diff --git a/tooling/collections/__tests__/update.patternFlyDocs.test.ts b/tooling/collections/__tests__/update.patternFlyDocs.test.ts new file mode 100644 index 00000000..5a7746bb --- /dev/null +++ b/tooling/collections/__tests__/update.patternFlyDocs.test.ts @@ -0,0 +1,455 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import { type PatternFlyMcpDocsCatalog } from '../../../src/docs.embedded'; +import { + DEFAULT_DOCS_FILENAME, + logDiffReport, + run +} from '../update.patternFlyDocs'; +import { fetchLatestRepoHashes, getSrcPath, writeJsonCollection } from '../helpers'; +import { saveCsvReport } from '../csv'; +import { findApiRedundantDocs, type DocsDiffResult } from '../update.patternFlyDocsHelpers'; + +jest.mock('node:fs', () => ({ + ...jest.requireActual('node:fs'), + existsSync: jest.fn((path: string) => (typeof path === 'string' && path.startsWith('/mock/')) || jest.requireActual('node:fs').existsSync(path)) +})); + +jest.mock('node:fs/promises', () => ({ + ...jest.requireActual('node:fs/promises'), + readFile: jest.fn() +})); + +jest.mock('../helpers', () => ({ + ...jest.requireActual('../helpers'), + fetchLatestRepoHashes: jest.fn(), + writeJsonCollection: jest.fn() +})); + +jest.mock('../csv', () => ({ + ...jest.requireActual('../csv'), + saveCsvReport: jest.fn() +})); + +const DOCS_PATH = getSrcPath(DEFAULT_DOCS_FILENAME); +const mockReadFile = readFile as jest.MockedFunction; +const mockWriteJsonCollection = writeJsonCollection as jest.MockedFunction; + +describe('collection.patternFlyDocs', () => { + it('should export the run function for programmatic invocation', () => { + expect(typeof run).toBe('function'); + }); + + it('should have a generated documentation manifest file', () => { + expect(existsSync(DOCS_PATH)).toBe(true); + }); + + it('should have a consistent JSON schema and correct metadata counts', () => { + const raw = readFileSync(DOCS_PATH, 'utf-8'); + const collection: PatternFlyMcpDocsCatalog = JSON.parse(raw); + + expect(collection).toMatchObject({ + version: expect.any(String), + generated: expect.any(String), + meta: { + totalEntries: expect.any(Number), + totalDocs: expect.any(Number), + source: expect.any(String) + }, + docs: expect.any(Object) + }); + + const totalEntries = Object.keys(collection.docs).length; + const totalDocs = Object.values(collection.docs).reduce((acc, arr) => acc + arr.length, 0); + + expect(collection.meta.totalEntries).toBe(totalEntries); + expect(collection.meta.totalDocs).toBe(totalDocs); + }); + + it('should have records that are properly formatted with key properties', () => { + const raw = readFileSync(DOCS_PATH, 'utf-8'); + const collection: PatternFlyMcpDocsCatalog = JSON.parse(raw); + const sample = Object.values(collection.docs).flatMap(docs => docs).slice(0, 50); + + expect(sample.length).toBeGreaterThan(0); + for (const record of sample) { + expect(typeof record.displayName).toBe('string'); + expect(typeof record.description).toBe('string'); + expect(typeof record.pathSlug).toBe('string'); + expect(typeof record.section).toBe('string'); + expect(typeof record.category).toBe('string'); + expect(typeof record.source).toBe('string'); + expect(typeof record.path).toBe('string'); + expect(typeof record.version).toBe('string'); + + expect(record.path.startsWith('http')).toBe(true); + } + }); +}); + +describe('logDiffReport', () => { + let consoleSpy: jest.SpyInstance; + + beforeEach(() => { + consoleSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + consoleSpy.mockRestore(); + }); + + it('should log structured diff summary with formatted additions, removals, and modifications', () => { + const sampleDiff: DocsDiffResult = { + added: [ + { + category: 'Components', + record: { + displayName: 'Button', + pathSlug: 'button', + path: 'https://raw.githubusercontent.com/patternfly/patternfly-react/sha/button.md', + category: 'react', + description: '', + section: 'components', + source: 'github', + version: 'v6' + } + } + ], + removed: [ + { + category: 'AI', + record: { + displayName: 'Table Rules', + pathSlug: 'table-rules', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/sha/table.md', + category: 'AI', + description: '', + section: 'guidelines', + source: 'github', + version: 'v6' + }, + reason: 'superseded by API collection', + details: 'v6/AI/development-guidelines_table/text' + } + ], + modified: [ + { + category: 'Alert', + record: { + displayName: 'Alert', + pathSlug: 'alert', + path: 'https://raw.githubusercontent.com/patternfly/patternfly-org/new/alert.md', + category: 'react', + description: '', + section: 'components', + source: 'github', + version: 'v6' + }, + reasons: ['hash update (old -> new)'] + } + ], + unchanged: [] + }; + + logDiffReport(sampleDiff); + + expect(consoleSpy).toHaveBeenCalledWith('\nšŸ“Š PatternFly Docs Collection Diff Report:'); + }); +}); + +describe('run', () => { + let consoleLogSpy: jest.SpyInstance; + let consoleWarnSpy: jest.SpyInstance; + + // Mock catalog with both standard component docs and AI marketplace docs + const mockCatalog: PatternFlyMcpDocsCatalog = { + version: '1', + generated: '2026-01-01T00:00:00.000Z', + meta: { totalEntries: 1, totalDocs: 1, source: 'patternfly-mcp' }, + docs: { + Button: [ + { + displayName: 'Button', + pathSlug: 'button', + path: 'https://raw.githubusercontent.com/patternfly/patternfly-react/1111111111111111111111111111111111111111/button.md', + version: 'v6', + category: 'react', + description: '', + section: 'components', + source: 'github' + } + ], + AIHelpers: [ + { + displayName: 'AI Helpers README', + pathSlug: 'ai-helpers-readme', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/1111111111111111111111111111111111111111/README.md', + version: 'v6', + category: 'AI', + description: 'Agent marketplace description', + section: 'guidelines', + source: 'github' + } + ] + } + }; + + const mockApiCollection = { + records: [ + { p: 'v6/AI/ai-assisted-development_marketplace/text', q: 1.0 } + ] + }; + + beforeEach(() => { + jest.clearAllMocks(); + consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}); + + mockReadFile.mockResolvedValue(JSON.stringify(mockCatalog)); + (fetchLatestRepoHashes as jest.MockedFunction).mockResolvedValue( + new Map([['patternfly/patternfly-react', '2222222222222222222222222222222222222222']]) + ); + mockWriteJsonCollection.mockResolvedValue({ + jsonContent: '{}', + sizeKb: '1.0', + durationSec: '0.1' + }); + (saveCsvReport as jest.MockedFunction).mockResolvedValue(undefined); + }); + + afterEach(() => { + consoleLogSpy.mockRestore(); + consoleWarnSpy.mockRestore(); + }); + + it('should default pruneApiOverlap to false and preserve overlapping documentation records', async () => { + const diff = await run({ docsPath: '/mock/docs.json', outputCsv: false }); + + expect(diff).toMatchObject({ + added: expect.any(Array), + removed: expect.any(Array), + modified: expect.any(Array), + unchanged: expect.any(Array) + }); + + // Overlap pruning is disabled by default: no records removed + expect(diff.removed).toHaveLength(0); + + // API file read should not have been attempted + expect(mockReadFile).toHaveBeenCalledTimes(1); + expect(mockReadFile).toHaveBeenCalledWith('/mock/docs.json', 'utf-8'); + }); + + it('should prune API redundant records when pruneApiOverlap is explicitly enabled', async () => { + mockReadFile + .mockResolvedValueOnce(JSON.stringify(mockCatalog)) + .mockResolvedValueOnce(JSON.stringify(mockApiCollection)); + + const diff = await run({ + docsPath: '/mock/docs.json', + apiPath: '/mock/api.json', + pruneApiOverlap: true, + updateHashes: false, + outputCsv: false + }); + + expect(mockReadFile).toHaveBeenCalledWith('/mock/docs.json', 'utf-8'); + expect(mockReadFile).toHaveBeenCalledWith('/mock/api.json', 'utf-8'); + + // ai-helpers-readme should be identified and removed as superseded + expect(diff.removed).toHaveLength(1); + expect(diff.removed[0]?.record.pathSlug).toBe('ai-helpers-readme'); + expect(diff.removed[0]?.reason).toBe('superseded by API collection'); + }); + + it('should preserve docs and skip API reading when pruneApiOverlap is explicitly false', async () => { + const diff = await run({ + docsPath: '/mock/docs.json', + apiPath: '/mock/api.json', + pruneApiOverlap: false, + updateHashes: false, + outputCsv: false + }); + + expect(mockReadFile).toHaveBeenCalledTimes(1); + expect(mockReadFile).toHaveBeenCalledWith('/mock/docs.json', 'utf-8'); + expect(diff.removed).toHaveLength(0); + }); + + it('should sync commit hashes by default when updateHashes is not disabled', async () => { + await run({ docsPath: '/mock/docs.json', outputCsv: false }); + + expect(fetchLatestRepoHashes).toHaveBeenCalledTimes(1); + expect(mockWriteJsonCollection).toHaveBeenCalled(); + + const savedCollection = mockWriteJsonCollection.mock.calls[0]?.[1] as PatternFlyMcpDocsCatalog; + + expect(savedCollection.docs.Button?.[0]?.path).toContain('2222222222222222222222222222222222222222'); + }); + + it('should bypass commit hash updates when updateHashes is false', async () => { + await run({ docsPath: '/mock/docs.json', updateHashes: false, outputCsv: false }); + + expect(fetchLatestRepoHashes).not.toHaveBeenCalled(); + + const savedCollection = mockWriteJsonCollection.mock.calls[0]?.[1] as PatternFlyMcpDocsCatalog; + + expect(savedCollection.docs.Button?.[0]?.path).toContain('1111111111111111111111111111111111111111'); + }); + + it('should generate and save CSV report by default or when outputCsv is true', async () => { + await run({ + docsPath: '/mock/docs.json', + csvOutputPath: '/mock/output.csv', + outputCsv: true + }); + + expect(saveCsvReport).toHaveBeenCalledTimes(1); + expect(saveCsvReport).toHaveBeenCalledWith('/mock/output.csv', expect.any(String)); + }); + + it('should skip CSV report generation when outputCsv is false', async () => { + await run({ docsPath: '/mock/docs.json', outputCsv: false }); + + expect(saveCsvReport).not.toHaveBeenCalled(); + }); + + it('should write JSON collection with correct stats and format flags', async () => { + await run({ + docsPath: '/mock/custom-docs.json', + isPrettyPrint: true, + outputCsv: false + }); + + expect(mockWriteJsonCollection).toHaveBeenCalledWith( + '/mock/custom-docs.json', + expect.any(Object), + expect.objectContaining({ + isPrettyPrint: true, + stats: expect.arrayContaining([ + { label: 'Total Categories', value: expect.any(Number) }, + { label: 'Total Documents', value: expect.any(Number) } + ]) + }) + ); + }); + + it('should safely handle and ignore malformed API JSON without failing the run', async () => { + mockReadFile + .mockResolvedValueOnce(JSON.stringify(mockCatalog)) + .mockResolvedValueOnce('invalid json'); + + const diff = await run({ pruneApiOverlap: true, updateHashes: false, outputCsv: false }); + + expect(diff).toBeDefined(); + expect(consoleWarnSpy).toHaveBeenCalledWith(expect.stringContaining('Could not parse API collection')); + }); +}); + +describe('API Deduplication Regression Guard (Deterministic Fixtures)', () => { + const fixtureDocs: PatternFlyMcpDocsCatalog = { + version: '1', + generated: '2026-01-01T00:00:00.000Z', + meta: { + totalEntries: 2, + totalDocs: 5, + source: 'patternfly-mcp' + }, + docs: { + AIHelpers: [ + { + displayName: 'AI Helpers README', + pathSlug: 'ai-helpers-readme', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/README.md', + version: 'v6', + category: 'AI', + description: 'Marketplace guide', + section: 'guidelines', + source: 'github' + }, + { + displayName: 'Table Rules', + pathSlug: 'table-rules', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/docs/components/data-display/table.md', + version: 'v6', + category: 'AI', + description: 'Table guidelines', + section: 'guidelines', + source: 'github' + }, + { + displayName: 'Unindexed Doc', + pathSlug: 'unindexed-guideline', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/docs/unindexed.md', + version: 'v6', + category: 'AI', + description: 'Unindexed document', + section: 'guidelines', + source: 'github' + }, + { + displayName: 'UXD Preserved Style Guide', + pathSlug: 'uxd-ai-helpers-style-guide', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/docs/style-guide.md', + version: 'v6', + category: 'AI', + description: 'UXD style guide', + section: 'guidelines', + source: 'github' + } + ], + Ecosystem: [ + { + displayName: 'CLI Guide', + pathSlug: 'patternfly-cli-readme', + path: 'https://raw.githubusercontent.com/patternfly/patternfly-cli/main/README.md', + version: 'v6', + category: 'tooling', + description: 'CLI tool documentation', + section: 'ecosystem', + source: 'github' + } + ] + } + }; + + const fixtureApiRecords = [ + { p: 'v6/AI/ai-assisted-development_marketplace/text', q: 1.0 }, + { p: 'v6/AI/development-guidelines_table/text', q: 1.0 } + ]; + + it('should detect redundant AI-helper entries matching qualifying high-quality API endpoints', () => { + const redundant = findApiRedundantDocs(fixtureDocs, fixtureApiRecords); + + expect(redundant).toHaveLength(2); + + const redundantSlugs = redundant.map(report => report.record.pathSlug); + + expect(redundantSlugs).toContain('ai-helpers-readme'); + expect(redundantSlugs).toContain('table-rules'); + + for (const item of redundant) { + expect(item.reason).toBe('superseded by API collection'); + expect(item.details).toBeDefined(); + } + }); + + it('should preserve root AIHelpers guides and ecosystem repositories without false positives', () => { + const redundant = findApiRedundantDocs(fixtureDocs, fixtureApiRecords); + const redundantSlugs = new Set(redundant.map(report => report.record.pathSlug)); + + // Unmatched AI helper documents must NOT be marked redundant + expect(redundantSlugs.has('unindexed-guideline')).toBe(false); + + // Negative boundary: uxd-ai-helpers-* prefix must be preserved + expect(redundantSlugs.has('uxd-ai-helpers-style-guide')).toBe(false); + + // Negative boundary: ecosystem repos must be preserved + expect(redundantSlugs.has('patternfly-cli-readme')).toBe(false); + }); + + it('should return empty array when API records list is empty or undefined', () => { + expect(findApiRedundantDocs(fixtureDocs, [])).toEqual([]); + expect(findApiRedundantDocs(fixtureDocs, undefined)).toEqual([]); + }); +}); diff --git a/tooling/collections/__tests__/update.patternFlyDocsHelpers.test.ts b/tooling/collections/__tests__/update.patternFlyDocsHelpers.test.ts new file mode 100644 index 00000000..b30b4154 --- /dev/null +++ b/tooling/collections/__tests__/update.patternFlyDocsHelpers.test.ts @@ -0,0 +1,509 @@ +import { type PatternFlyMcpDocsCatalog } from '../../../src/docs.embedded'; +import { + AI_GUIDELINE_ALIASES, + PINNED_HISTORICAL_REFS, + diffDocsManifests, + findApiRedundantDocs, + generateReportCsv, + recalculateManifestMetadata, + resolveApiEndpointForAiDoc +} from '../update.patternFlyDocsHelpers'; +import { verifyUrlReachability } from '../helpers'; + +describe('AI_GUIDELINE_ALIASES', () => { + it('should maintain guideline aliases', () => { + expect(AI_GUIDELINE_ALIASES['table-rules']).toBe('table'); + expect(AI_GUIDELINE_ALIASES['layout-components']).toBe('layout'); + expect(AI_GUIDELINE_ALIASES['development-rules']).toBe('overview'); + }); +}); + +describe('PINNED_HISTORICAL_REFS', () => { + it('should maintain declarative pinned historical hashes', () => { + expect(PINNED_HISTORICAL_REFS instanceof Set).toBe(true); + expect(PINNED_HISTORICAL_REFS.has('ec02b437ec72b6e4cc4e28524516288f4acf9fdf')).toBe(true); + expect(PINNED_HISTORICAL_REFS.has('e8cca17430a8ccb062ed1878073165417a081b34')).toBe(true); + }); +}); + +describe('resolveApiEndpointForAiDoc', () => { + const highQualityApi = [ + { p: 'v6/AI/ai-assisted-development_marketplace/text', q: 1 }, + { p: 'v6/AI/development-guidelines_overview/text', q: 1 }, + { p: 'v6/AI/development-guidelines_table/text', q: 1 }, + { p: 'v6/AI/development-guidelines_charts/text', q: 1 } + ]; + + it.each([ + { + description: 'README alias matching marketplace endpoint', + doc: { + displayName: 'AI Helpers README', + pathSlug: 'ai-helpers-readme', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/README.md' + }, + expected: 'v6/AI/ai-assisted-development_marketplace/text' + }, + { + description: 'contributing alias matching marketplace endpoint', + doc: { + displayName: 'AI Helpers Contributing', + pathSlug: 'ai-helpers-contributing', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/CONTRIBUTING.md' + }, + expected: 'v6/AI/ai-assisted-development_marketplace/text' + }, + { + description: 'table rules alias matching table endpoint', + doc: { + displayName: 'Table Rules', + pathSlug: 'table-rules', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/docs/components/data-display/table.md' + }, + expected: 'v6/AI/development-guidelines_table/text' + }, + { + description: 'charts token matching charts endpoint via fuzzy lookup', + doc: { + displayName: 'Charts Rules', + pathSlug: 'charts', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/docs/charts/README.md' + }, + expected: 'v6/AI/development-guidelines_charts/text' + } + ])('should resolve AI endpoint, $description', ({ doc, expected }) => { + const fullDoc = { + version: 'v6', + category: 'AI', + description: '', + section: '', + source: 'github' as const, + ...doc + }; + + expect(resolveApiEndpointForAiDoc(fullDoc, highQualityApi)).toBe(expected); + }); + + it('should return undefined when no qualifying API endpoint matches', () => { + const unindexedDoc = { + displayName: 'Unindexed Doc', + pathSlug: 'unindexed-guideline', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/docs/unindexed.md', + version: 'v6', + category: 'AI', + description: '', + section: '', + source: 'github' as const + }; + + expect(resolveApiEndpointForAiDoc(unindexedDoc, [])).toBeUndefined(); + expect(resolveApiEndpointForAiDoc(unindexedDoc, highQualityApi)).toBeUndefined(); + }); + + it('should return undefined when document is not an AI helper docs path', () => { + const nonDocsEntry = { + displayName: 'Generic Doc', + pathSlug: 'generic-doc', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/scripts/run.md', + version: 'v6', + category: 'AI', + description: '', + section: '', + source: 'github' as const + }; + + expect(resolveApiEndpointForAiDoc(nonDocsEntry, highQualityApi)).toBeUndefined(); + }); +}); + +describe('findApiRedundantDocs', () => { + const sampleCatalog: PatternFlyMcpDocsCatalog = { + version: '1', + generated: '2026-01-01T00:00:00.000Z', + meta: { totalEntries: 3, totalDocs: 5, source: 'patternfly-mcp' }, + docs: { + AIHelpers: [ + { + displayName: 'AI Helpers README', + pathSlug: 'ai-helpers-readme', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/README.md', + version: 'v6', + category: 'AI', + description: '', + section: '', + source: 'github' + }, + { + displayName: 'Table Rules', + pathSlug: 'table-rules', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/docs/components/data-display/table.md', + version: 'v6', + category: 'AI', + description: '', + section: '', + source: 'github' + }, + { + displayName: 'UXD Preserved Guide', + pathSlug: 'uxd-ai-helpers-guideline', + path: 'https://raw.githubusercontent.com/rh-uxd/ai-helpers/main/docs/guidelines.md', + version: 'v6', + category: 'AI', + description: '', + section: '', + source: 'github' + } + ], + Ecosystem: [ + { + displayName: 'CLI Guide', + pathSlug: 'patternfly-cli-readme', + path: 'https://raw.githubusercontent.com/patternfly/patternfly-cli/main/README.md', + version: 'v6', + category: 'tooling', + description: '', + section: '', + source: 'github' + } + ] + } + }; + + const apiRecords = [ + { p: 'v6/AI/ai-assisted-development_marketplace/text', q: 1.0 }, + { p: 'v6/AI/development-guidelines_table/text', q: 1.0 }, + { p: 'v6/AI/development-guidelines_low_quality/text', q: 0.8 } + ]; + + it('should detect redundant documents when qualifying high-quality API endpoints match', () => { + const redundant = findApiRedundantDocs(sampleCatalog, apiRecords); + const redundantSlugs = redundant.map(item => item.record.pathSlug); + + expect(redundant).toHaveLength(2); + expect(redundantSlugs).toContain('ai-helpers-readme'); + expect(redundantSlugs).toContain('table-rules'); + }); + + it('should strictly preserve negative control boundaries and ecosystem repositories', () => { + const redundant = findApiRedundantDocs(sampleCatalog, apiRecords); + const removedSlugs = new Set(redundant.map(item => item.record.pathSlug)); + + expect(removedSlugs.has('uxd-ai-helpers-guideline')).toBe(false); + expect(removedSlugs.has('patternfly-cli-readme')).toBe(false); + }); + + it('should return empty array when API records list is empty or undefined', () => { + expect(findApiRedundantDocs(sampleCatalog, [])).toEqual([]); + expect(findApiRedundantDocs(sampleCatalog, undefined)).toEqual([]); + }); +}); + +describe('generateReportCsv', () => { + it('should produce a full structured CSV report for added, removed, modified, and unchanged doc entries', () => { + const diff = { + added: [ + { + category: 'NewCat', + record: { + displayName: 'New Doc', + description: 'New Description', + pathSlug: 'new-doc', + section: 'components', + category: 'react', + source: 'github', + path: 'https://raw.githubusercontent.com/patternfly/patternfly-react/831257aa7c49c3238e0f7afbb7cf219c62cd9e23/README.md', + version: 'v6' + }, + reason: 'new upstream document', + details: 'Discovered in manifest update' + } + ], + removed: [ + { + category: 'React', + record: { + displayName: 'Table Rules', + description: 'Old Table Rules', + pathSlug: 'table-rules', + section: 'components', + category: 'react', + source: 'github', + path: 'https://raw.githubusercontent.com/patternfly/ai-helpers/f7f8160c3f28b0bc7f64181d9466a425ac8329fc/docs/components/data-display/table.md', + version: 'v6' + }, + reason: 'superseded by API collection', + details: 'v6/AI/development-guidelines_table/text' + } + ], + modified: [ + { + category: 'Button', + record: { + displayName: 'Button Design', + description: 'Button design guidelines', + pathSlug: 'button-design', + section: 'components', + category: 'design-guidelines', + source: 'github', + path: 'https://raw.githubusercontent.com/patternfly/patternfly-org/540bb0d31cb18670dd02857f80aa8b444fed9be9/Button.md', + version: 'v6' + }, + reasons: ['hash update (9577561 -> 540bb0d)'], + previousHash: '957756128e8ddfc4be5db49e72312a2c43b9d220', + newHash: '540bb0d31cb18670dd02857f80aa8b444fed9be9' + } + ], + unchanged: [ + { + category: 'Alert', + record: { + displayName: 'Alert Design', + description: 'Alert design guidelines', + pathSlug: 'alert-design', + section: 'components', + category: 'design-guidelines', + source: 'github', + path: 'https://raw.githubusercontent.com/patternfly/patternfly-org/540bb0d31cb18670dd02857f80aa8b444fed9be9/Alert.md', + version: 'v6' + } + } + ] + }; + + const csv = generateReportCsv({ diff }); + const lines = csv.trim().split('\n'); + + expect(lines[0]).toBe('status,category,name,pathSlug,path,previousHash,newHash,reason,details'); + expect(lines.some(line => line.startsWith('ADDED,NewCat,New Doc') && line.includes('new upstream document'))).toBe(true); + expect(lines.some(line => line.startsWith('REMOVED,React,Table Rules') && line.includes('superseded by API collection'))).toBe(true); + expect(lines.some(line => line.startsWith('MODIFIED,Button,Button Design') && line.includes('hash update'))).toBe(true); + expect(lines.some(line => line.startsWith('UNCHANGED,Alert,Alert Design'))).toBe(true); + }); +}); + +describe('diffDocsManifests', () => { + it('should diff manifests and categorize added, removed, modified, and unchanged entries', () => { + const oldCollection: PatternFlyMcpDocsCatalog = { + meta: { totalEntries: 2, totalDocs: 2, source: 'test' }, + docs: { + Cat1: [ + { + displayName: 'Doc A', + description: 'Desc A', + pathSlug: 'doc-a', + section: 'sec', + category: 'cat', + source: 'github', + path: 'https://raw.githubusercontent.com/owner/repo/hash1111111111111111111111111111111111111111/a.md', + version: 'v6' + } + ], + Cat2: [ + { + displayName: 'Doc B', + description: 'Desc B', + pathSlug: 'doc-b', + section: 'sec', + category: 'cat', + source: 'github', + path: 'https://raw.githubusercontent.com/owner/repo/hash1111111111111111111111111111111111111111/b.md', + version: 'v6' + } + ] + } + }; + + const newCollection: PatternFlyMcpDocsCatalog = { + meta: { totalEntries: 2, totalDocs: 2, source: 'test' }, + docs: { + Cat1: [ + { + displayName: 'Doc A', + description: 'Desc A', + pathSlug: 'doc-a', + section: 'sec', + category: 'cat', + source: 'github', + path: 'https://raw.githubusercontent.com/owner/repo/hash2222222222222222222222222222222222222222/a.md', + version: 'v6' + } + ], + Cat3: [ + { + displayName: 'Doc C', + description: 'Desc C', + pathSlug: 'doc-c', + section: 'sec', + category: 'cat', + source: 'github', + path: 'https://raw.githubusercontent.com/owner/repo/hash2222222222222222222222222222222222222222/c.md', + version: 'v6' + } + ] + } + }; + + const diff = diffDocsManifests(oldCollection, newCollection); + + expect(diff.modified.length).toBe(1); + expect(diff.modified[0]?.record.displayName).toBe('Doc A'); + expect(diff.removed.length).toBe(1); + expect(diff.removed[0]?.record.displayName).toBe('Doc B'); + expect(diff.added.length).toBe(1); + expect(diff.added[0]?.record.displayName).toBe('Doc C'); + }); +}); + +describe('recalculateManifestMetadata', () => { + const sampleCollection: PatternFlyMcpDocsCatalog = { + version: '1', + generated: '2026-01-01T00:00:00.000Z', + meta: { + totalEntries: 2, + totalDocs: 2, + source: 'patternfly-mcp' + }, + docs: { + Button: [ + { + displayName: 'Button Design', + description: 'Button design guidelines', + pathSlug: 'button-design', + section: 'components', + category: 'design-guidelines', + source: 'github', + path: 'https://raw.githubusercontent.com/patternfly/patternfly-org/540bb0d31cb18670dd02857f80aa8b444fed9be9/Button.md', + version: 'v6' + } + ], + Alert: [ + { + displayName: 'Alert Design', + description: 'Alert design guidelines', + pathSlug: 'alert-design', + section: 'components', + category: 'design-guidelines', + source: 'github', + path: 'https://raw.githubusercontent.com/patternfly/patternfly-org/540bb0d31cb18670dd02857f80aa8b444fed9be9/Alert.md', + version: 'v6' + } + ] + } + }; + + it('should recalculate manifest metadata totalEntries and totalDocs accurately', async () => { + const latestHashes = new Map([ + ['patternfly/patternfly-org', '3333333333333333333333333333333333333333'] + ]); + + const { collection: updated } = await recalculateManifestMetadata(sampleCollection, { latestHashes }); + + expect(updated.meta.totalEntries).toBe(2); + expect(updated.meta.totalDocs).toBe(2); + expect(updated.docs.Button?.[0]?.path).toContain('3333333333333333333333333333333333333333'); + expect(updated.docs.Alert?.[0]?.path).toContain('3333333333333333333333333333333333333333'); + }); + + it('should recalculate metadata and update commit hashes for unpinned repositories', async () => { + const hashes = new Map([ + ['patternfly/patternfly-org', '1111111111111111111111111111111111111111'] + ]); + + const { collection: result } = await recalculateManifestMetadata(sampleCollection, { + latestHashes: hashes + }); + + expect(result.docs['Button']?.[0]?.path).toBe( + 'https://raw.githubusercontent.com/patternfly/patternfly-org/1111111111111111111111111111111111111111/Button.md' + ); + expect(result.docs['Alert']?.[0]?.path).toBe( + 'https://raw.githubusercontent.com/patternfly/patternfly-org/1111111111111111111111111111111111111111/Alert.md' + ); + }); + + it('should prune redundant AI helper entries and excludes empty categories from output', async () => { + const redundantReports = [ + { + category: 'Overview', + record: { + displayName: 'Overview Guide', + description: 'Overview guide', + pathSlug: 'overview-guide', + section: 'components', + category: 'react', + source: 'github', + path: 'https://raw.githubusercontent.com/patternfly/ai-helpers/main/docs/overview.md', + version: 'v6' + }, + reason: 'superseded by API collection' + } + ]; + + const collectionWithOverview: PatternFlyMcpDocsCatalog = { + ...sampleCollection, + docs: { + ...sampleCollection.docs, + Overview: [ + redundantReports[0]!.record + ] + } + }; + + const { collection: result } = await recalculateManifestMetadata(collectionWithOverview, { + redundantRecords: redundantReports + }); + + expect(result.docs['Overview']).toBeUndefined(); + expect(result.docs['Button']).toBeDefined(); + expect(result.docs['Alert']).toBeDefined(); + expect(result.meta.totalEntries).toBe(2); + expect(result.meta.totalDocs).toBe(2); + }); + + it.each([ + { + description: 'prunes broken URLs (HTTP 404)', + latestHashes: undefined, + mockReachable: (url: string) => !url.includes('Button.md'), + expectedPaths: [ + 'https://raw.githubusercontent.com/patternfly/patternfly-org/540bb0d31cb18670dd02857f80aa8b444fed9be9/Alert.md' + ], + expectedRemovals: ['broken upstream link (HTTP 404)'] + }, + { + description: 'falls back to previous URL when bumped hash drifts/404s', + latestHashes: new Map([['patternfly/patternfly-org', '1111111111111111111111111111111111111111']]), + mockReachable: (url: string) => url.includes('540bb0d31cb18670dd02857f80aa8b444fed9be9'), + expectedPaths: [ + 'https://raw.githubusercontent.com/patternfly/patternfly-org/540bb0d31cb18670dd02857f80aa8b444fed9be9/Alert.md', + 'https://raw.githubusercontent.com/patternfly/patternfly-org/540bb0d31cb18670dd02857f80aa8b444fed9be9/Button.md' + ], + expectedRemovals: [] + }, + { + description: 'updates commit hash when new URL is reachable', + latestHashes: new Map([['patternfly/patternfly-org', '1111111111111111111111111111111111111111']]), + mockReachable: () => true, + expectedPaths: [ + 'https://raw.githubusercontent.com/patternfly/patternfly-org/1111111111111111111111111111111111111111/Alert.md', + 'https://raw.githubusercontent.com/patternfly/patternfly-org/1111111111111111111111111111111111111111/Button.md' + ], + expectedRemovals: [] + } + ])('should recalculate manifest metadata, $description', async ({ latestHashes, mockReachable, expectedPaths, expectedRemovals }) => { + const reachabilitySpy = jest.spyOn(verifyUrlReachability, 'memo').mockImplementation(async (url: string) => mockReachable(url)); + + const { collection: result, reachabilityRemovals } = await recalculateManifestMetadata(sampleCollection, { + latestHashes, + verifyReachability: true + } as any); + + expect(reachabilityRemovals.map(removal => removal.reason)).toEqual(expectedRemovals); + + const paths = Object.entries(result.docs).map(([_key, categoryDocs]) => categoryDocs[0]?.path); + + expect(paths).toEqual(expect.arrayContaining(expectedPaths)); + + reachabilitySpy.mockRestore(); + }); +}); diff --git a/tooling/collections/helpers.ts b/tooling/collections/helpers.ts index ae918e14..7d9af90d 100644 --- a/tooling/collections/helpers.ts +++ b/tooling/collections/helpers.ts @@ -1,6 +1,278 @@ +import { execSync } from 'node:child_process'; import { writeFile } from 'node:fs/promises'; import { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { memo } from '../../src/server.caching'; +import { type PatternFlyMcpDocsCatalog } from '../../src/docs.embedded'; + +/** + * Information extracted from a raw GitHub documentation URL. + */ +interface GitHubUrlInfo { + owner: string; + repo: string; + ref: string; + filePath: string; +} + +/** + * Repository tracking definition for upstream release queries. + */ +interface TrackedRepository { + owner: string; + repo: string; + branch?: string | undefined; +} + +/** + * Extract commit hash or ref from a raw GitHub documentation URL. + * + * @param url - Raw GitHub URL + * @returns Hash/ref string or undefined if not a recognized GitHub raw URL + */ +const extractCommitHash = (url: string): string | undefined => { + if (!url || typeof url !== 'string') { + return undefined; + } + + const match = url.match(/^https:\/\/raw\.githubusercontent\.com\/[^/]+\/[^/]+\/([^/]+)\//); + + return match && match[1] ? match[1] : undefined; +}; + +/** + * Extract structured repository and path information from a raw GitHub URL. + * + * @param url - Raw GitHub URL + * @returns GitHubUrlInfo or undefined if not a recognized raw URL + */ +const extractRepoInfo = (url: string): GitHubUrlInfo | undefined => { + if (!url || typeof url !== 'string') { + return undefined; + } + + const match = url.match(/^https:\/\/raw\.githubusercontent\.com\/([^/]+)\/([^/]+)\/([^/]+)\/(.+)$/); + + if (!match || !match[1] || !match[2] || !match[3] || !match[4]) { + return undefined; + } + + return { + owner: match[1], + repo: match[2], + ref: match[3], + filePath: match[4] + }; +}; + +/** + * Memoized version of extractRepoInfo. + */ +extractRepoInfo.memo = memo(extractRepoInfo, { + cacheLimit: 500, + keyHash: args => args[0] +}); + +/** + * Dynamically extract unique repositories referenced across all documents in a collection. + * + * @param collection - Documentation collection to inspect + * @param [defaultBranch] - Optional explicit tracking branch override + * @returns Array of unique TrackedRepository definitions + */ +const extractTrackedReposFromCollection = ( + collection: PatternFlyMcpDocsCatalog, + defaultBranch?: string +): TrackedRepository[] => { + const uniqueRepos = new Map(); + + for (const entries of Object.values(collection?.docs || {})) { + for (const doc of entries) { + if (!doc?.path || typeof doc.path !== 'string') { + continue; + } + + const info = extractRepoInfo(doc.path); + + if (info && info.owner && info.repo) { + const key = `${info.owner}/${info.repo}`.toLowerCase(); + + if (!uniqueRepos.has(key)) { + uniqueRepos.set(key, { owner: info.owner, repo: info.repo, branch: defaultBranch }); + } + } + } + } + + return Array.from(uniqueRepos.values()); +}; + +/** + * Fallback helper to query HEAD commit SHA using `git ls-remote --symref`. + * + * @param owner - Repository owner/organization + * @param repo - Repository name + * @returns Commit SHA or undefined if git is unavailable or command fails + */ +const fetchCommitViaGitLsRemote = (owner: string, repo: string): string | undefined => { + try { + const remoteUrl = `https://github.com/${owner}/${repo}.git`; + const output = execSync(`git ls-remote --symref ${remoteUrl} HEAD`, { + encoding: 'utf-8', + timeout: 10_000, + stdio: ['ignore', 'pipe', 'ignore'] + }); + + const shaMatch = output.match(/^([0-9a-f]{40})\s+HEAD/m); + + if (shaMatch && shaMatch[1]) { + return shaMatch[1]; + } + + const fallbackMatch = output.match(/([0-9a-f]{40})/); + + return fallbackMatch && fallbackMatch[1] ? fallbackMatch[1] : undefined; + } catch { + return undefined; + } +}; + +/** + * Internal helper to perform a probe request with guaranteed timer cleanup. + * + * @param url - Target URL to probe + * @param method - HTTP method to use ('HEAD' | 'GET') + * @param timeoutMs - Timeout in milliseconds + * @returns Promise resolving to Response + */ +const probeUrl = async (url: string, method: 'HEAD' | 'GET', timeoutMs: number): Promise => { + const controller = new AbortController(); + const timeoutId = setTimeout(() => { + controller.abort(); + }, timeoutMs); + + try { + return await fetch(url, { + method, + headers: { 'User-Agent': 'patternfly-mcp-audit' }, + signal: controller.signal + }); + } finally { + clearTimeout(timeoutId); + } +}; + +/** + * Probe URL reachability using HTTP HEAD / GET request. + * + * @param url - Target URL to probe + * @param [timeoutMs=5000] - Timeout in milliseconds + * @returns Promise resolving to true if status is 2xx, false otherwise + */ +const verifyUrlReachability = async (url: string, timeoutMs = 5000): Promise => { + try { + const response = await probeUrl(url, 'HEAD', timeoutMs); + + if (response.ok) { + return true; + } + + // Fallback to GET for hosts that reject HEAD requests + if (response.status === 405 || response.status === 403) { + const getResponse = await probeUrl(url, 'GET', timeoutMs); + + return getResponse.ok; + } + + return false; + } catch { + return false; + } +}; + +/** + * Memoized version of verifyUrlReachability. + */ +verifyUrlReachability.memo = memo(verifyUrlReachability, { + cacheLimit: 200, + keyHash: args => `${args[0]}:${args[1] || 5000}` +}); + +/** + * Resolve latest commit SHA for an individual repository via GitHub API or git ls-remote fallback. + * + * @param owner - Repository owner + * @param repo - Repository name + * @param [branch] - Optional target branch + * @returns Commit SHA or undefined if unresolved + */ +const fetchRepoCommit = async ( + owner: string, + repo: string, + branch?: string +): Promise => { + let sha: string | undefined = undefined; + + try { + const url = branch + ? `https://api.github.com/repos/${owner}/${repo}/commits/${branch}` + : `https://api.github.com/repos/${owner}/${repo}/commits`; + + const response = await fetch(url, { + headers: { + 'User-Agent': 'patternfly-mcp', + Accept: 'application/vnd.github.v3+json' + } + }); + + if (response.ok) { + const data = await response.json(); + + if (Array.isArray(data) && data[0]?.sha) { + sha = data[0].sha; + } else if (data && typeof data === 'object' && 'sha' in data && typeof data.sha === 'string') { + sha = data.sha; + } + } + } catch { + // Fallback on network/API failure + } + + return sha || fetchCommitViaGitLsRemote(owner, repo); +}; + +/** + * Memoized version of fetchRepoCommit. + */ +fetchRepoCommit.memo = memo(fetchRepoCommit, { + cacheLimit: 50, + expire: 60_000, + keyHash: args => `${args[0]}/${args[1]}:${args[2] || 'default'}` +}); + +/** + * Fetch latest commit hashes for tracked repositories via GitHub API with git ls-remote fallback. + * + * @param repos - Repositories to query + * @returns Map of "owner/repo" and "repo" to commit SHA + */ +const fetchLatestRepoHashes = async ( + repos: TrackedRepository[] = [] +): Promise> => { + const hashes = new Map(); + + for (const { owner, repo, branch } of repos) { + const key = `${owner}/${repo}`; + const sha = await fetchRepoCommit.memo(owner, repo, branch); + + if (sha) { + hashes.set(key, sha); + hashes.set(repo, sha); + } + } + + return hashes; +}; /** * Options for writeJsonCollection. @@ -90,9 +362,19 @@ const runUpdateTask = async ( }; export { + extractCommitHash, + extractRepoInfo, + extractTrackedReposFromCollection, + fetchCommitViaGitLsRemote, + fetchRepoCommit, + fetchLatestRepoHashes, getReportsPath, getSrcPath, + probeUrl, runUpdateTask, + verifyUrlReachability, writeJsonCollection, + type GitHubUrlInfo, + type TrackedRepository, type WriteJsonCollectionOptions }; diff --git a/tooling/collections/update.patternFlyDocs.ts b/tooling/collections/update.patternFlyDocs.ts new file mode 100644 index 00000000..ab9db3cd --- /dev/null +++ b/tooling/collections/update.patternFlyDocs.ts @@ -0,0 +1,179 @@ +import { existsSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import { type PatternFlyMcpDocsCatalog } from '../../src/docs.embedded'; +import { + getDefaultReportPath, + saveCsvReport +} from './csv'; +import { + extractTrackedReposFromCollection, + fetchLatestRepoHashes, + getSrcPath, + runUpdateTask, + writeJsonCollection +} from './helpers'; +import { printDiffSummary } from './summary'; +import { + diffDocsManifests, + findApiRedundantDocs, + generateReportCsv, + recalculateManifestMetadata, + type ApiCollectionRecordRef, + type DocsDiffResult +} from './update.patternFlyDocsHelpers'; + +/** + * Options for running the documentation manifest collection update. + */ +interface UpdateDocsOptions { + docsFilename?: string | undefined; + apiFilename?: string | undefined; + reportFilename?: string | undefined; + docsPath?: string | undefined; + apiPath?: string | undefined; + csvOutputPath?: string | undefined; + pruneApiOverlap?: boolean | undefined; + verifyReachability?: boolean | undefined; + updateHashes?: boolean | undefined; + isPrettyPrint?: boolean | undefined; + outputCsv?: boolean | undefined; +} + +/** + * Configurable default filename for the documentation collection. + */ +const DEFAULT_DOCS_FILENAME = 'docs.json'; + +/** + * Configurable default filename for the API collection. + */ +const DEFAULT_API_FILENAME = 'collection.patternFlyApi.json'; + +/** + * Configurable default filename for the documentation report. + */ +const DEFAULT_DOCS_REPORT_FILENAME = 'collection.patternFlyDocs.report.csv'; + +/** + * Log a structured diff summary of the documentation changes to the console. + * + * @param diff - Complete diff result + */ +const logDiffReport = (diff: DocsDiffResult) => { + printDiffSummary(diff, { + title: 'PatternFly Docs Collection Diff Report', + formatAdded: item => `[${item.category}] ${item.record.displayName} (${item.record.pathSlug})`, + formatRemoved: item => `[${item.category}] ${item.record.displayName} (${item.record.pathSlug}) [Reason: ${item.reason}${item.details ? ` ā€” ${item.details}` : ''}]`, + formatModified: item => `[${item.category}] ${item.record.displayName} [${item.reasons.join(', ')}]` + }); +}; + +/** + * Update the PatternFly Docs manifest collection, deduplicating against the API seed and syncing repository SHAs. + * + * @param [options] - Optional execution options. + * @param [options.docsFilename=DEFAULT_DOCS_FILENAME] - Target documentation filename. + * @param [options.apiFilename=DEFAULT_API_FILENAME] - Target API filename for deduplication. + * @param [options.reportFilename=DEFAULT_DOCS_REPORT_FILENAME] - Target CSV report filename. + * @param [options.docsPath] - Fully resolved path to write the documentation collection. + * @param [options.apiPath] - Fully resolved path to read the API collection. + * @param [options.csvOutputPath] - Fully resolved path to write the CSV report. + * @param [options.pruneApiOverlap=false] - Whether to prune duplicate docs that exist in API collection. + * @param [options.verifyReachability=false] - Whether to probe URLs for reachability. + * @param [options.updateHashes=true] - Whether to sync repository commit SHAs. + * @param [options.isPrettyPrint=true] - Whether to pretty-print the JSON output. + * @param [options.outputCsv=true] - Whether to generate and save CSV report. + * @returns Promise resolving to the diff result + */ +const run = async ({ + docsFilename = DEFAULT_DOCS_FILENAME, + apiFilename = DEFAULT_API_FILENAME, + reportFilename = DEFAULT_DOCS_REPORT_FILENAME, + docsPath = getSrcPath(docsFilename), + apiPath = getSrcPath(apiFilename), + csvOutputPath = getDefaultReportPath(reportFilename), + pruneApiOverlap = false, + verifyReachability = false, + updateHashes = true, + isPrettyPrint = true, + outputCsv = true +}: UpdateDocsOptions = {}): Promise => { + console.log('šŸš€ Updating PatternFly Docs manifest collection...'); + const startTime = Date.now(); + + // 1. Read the existing documentation collection + const rawDocs = await readFile(docsPath, 'utf-8'); + const oldCollection: PatternFlyMcpDocsCatalog = JSON.parse(rawDocs); + + // 2. Read API collection for deduplication if enabled and present + let apiRecords: ApiCollectionRecordRef[] = []; + + if (pruneApiOverlap && existsSync(apiPath)) { + try { + const rawApi = await readFile(apiPath, 'utf-8'); + const apiCollection = JSON.parse(rawApi); + + apiRecords = apiCollection.records || []; + } catch { + console.warn(`āš ļø Could not parse API collection at ${apiPath}. Skipping API deduplication.`); + } + } + + // 3. Identify redundant records superseded by API collection + const redundantRecords = pruneApiOverlap ? findApiRedundantDocs(oldCollection, apiRecords) : []; + + // 4. Fetch latest commit SHAs for tracked upstream repositories if enabled + let latestHashes = new Map(); + + if (updateHashes) { + const trackedRepos = extractTrackedReposFromCollection(oldCollection); + + latestHashes = await fetchLatestRepoHashes(trackedRepos); + } + + // 5. Build an updated collection and recalculate manifest metadata + const { collection: updatedCollection, reachabilityRemovals } = await recalculateManifestMetadata(oldCollection, { + redundantRecords, + latestHashes: updateHashes ? latestHashes : undefined, + verifyReachability + }); + + // 6. Write an updated documentation manifest + await writeJsonCollection(docsPath, updatedCollection, { + isPrettyPrint, + startTime, + stats: [ + { label: 'Total Categories', value: updatedCollection.meta.totalEntries }, + { label: 'Total Documents', value: updatedCollection.meta.totalDocs } + ] + }); + + // 7. Calculate diff and print console summary + const allRemovals = [...redundantRecords, ...reachabilityRemovals]; + const diff = diffDocsManifests(oldCollection, updatedCollection, allRemovals); + + logDiffReport(diff); + + // 8. Generate and save CSV report if requested + if (outputCsv) { + const csvContent = generateReportCsv({ diff }); + + await saveCsvReport(csvOutputPath, csvContent); + } + + return diff; +}; + +/** + * Direct execution when invoked via UPDATE_COLLECTIONS=true + */ +runUpdateTask('Docs collection', run); + +export { + DEFAULT_API_FILENAME, + DEFAULT_DOCS_FILENAME, + DEFAULT_DOCS_REPORT_FILENAME, + logDiffReport, + run, + type UpdateDocsOptions +}; diff --git a/tooling/collections/update.patternFlyDocsHelpers.ts b/tooling/collections/update.patternFlyDocsHelpers.ts new file mode 100644 index 00000000..4e3b3991 --- /dev/null +++ b/tooling/collections/update.patternFlyDocsHelpers.ts @@ -0,0 +1,545 @@ +import { + type PatternFlyMcpDocsCatalog, + type PatternFlyMcpDocsCatalogDoc, + type PatternFlyMcpDocsCatalogEntry +} from '../../src/docs.embedded'; +import { + extractCommitHash, + extractRepoInfo, + verifyUrlReachability +} from './helpers'; +import { generateDiffCsv } from './csv'; + +/** + * Report entry for an added document in the manifest. + */ +interface DocsAddedRecordReport { + category: string; + record: PatternFlyMcpDocsCatalogDoc; + reason?: string | undefined; + details?: string | undefined; +} + +/** + * Report entry for a removed document in the manifest. + */ +interface DocsRemovedRecordReport { + category: string; + record: PatternFlyMcpDocsCatalogDoc; + reason: string; + details?: string | undefined; +} + +/** + * Report entry for a modified document in the manifest. + */ +interface DocsModifiedRecordReport { + category: string; + record: PatternFlyMcpDocsCatalogDoc; + previousRecord?: PatternFlyMcpDocsCatalogDoc | undefined; + reasons: string[]; + previousHash?: string | undefined; + newHash?: string | undefined; +} + +/** + * Report entry for an unchanged document in the manifest. + */ +interface DocsUnchangedRecordReport { + category: string; + record: PatternFlyMcpDocsCatalogDoc; +} + +/** + * Complete diff result between two documentation manifests. + */ +interface DocsDiffResult { + added: DocsAddedRecordReport[]; + removed: DocsRemovedRecordReport[]; + modified: DocsModifiedRecordReport[]; + unchanged: DocsUnchangedRecordReport[]; +} + +/** + * Options for generating Docs collection CSV diff report. + */ +interface GenerateCsvReportOptions { + diff: DocsDiffResult; +} + +/** + * Minimal shape of an API embedded collection record for cross-referencing. + */ +interface ApiCollectionRecordRef { + p: string; + n?: string | undefined; + q?: number | undefined; +} + +/** + * Options for recalculating manifest metadata and transforming entries. + */ +interface RecalculateOptions { + redundantRecords?: DocsRemovedRecordReport[] | undefined; + latestHashes?: Map | undefined; + verifyReachability?: boolean | undefined; + probeConcurrency?: number | undefined; +} + +/** + * Result returned by asynchronous manifest recalculation. + */ +interface RecalculateManifestResult { + collection: PatternFlyMcpDocsCatalog; + reachabilityRemovals: DocsRemovedRecordReport[]; +} + +/** + * Concise mapping of non-standard slugs to their corresponding API endpoint paths. + */ +const AI_GUIDELINE_ALIASES: Readonly> = Object.freeze({ + 'development-rules': 'overview', + guidelines: 'overview', + 'table-rules': 'table', + 'layout-components': 'layout', + troubleshooting: 'common-issues' +}); + +/** + * Pinned historical commit SHAs preserved for legacy or static references. + */ +const PINNED_HISTORICAL_REFS = new Set([ + 'ec02b437ec72b6e4cc4e28524516288f4acf9fdf', // Legacy design guidelines + 'ce032cd16ddb90c540cb4f18c6830e190cd9e3e9', // Tooling reference + '402b3b0e7ed73cb2aa21531e0eab4216c2211212', // Elements reference + 'e8cca17430a8ccb062ed1878073165417a081b34' // AIHelpers pinned baseline +]); + +/** + * Resolve the matching API endpoint for an upstream AI helper document. + * + * @param doc - Document record to evaluate + * @param highQualityApi - List of API collection records meeting the quality threshold + * @returns Matched API endpoint path or undefined if not superseded + */ +const resolveApiEndpointForAiDoc = ( + doc: PatternFlyMcpDocsCatalogDoc, + highQualityApi: ApiCollectionRecordRef[] +): string | undefined => { + const slug = doc.pathSlug || ''; + + // 1. Root ai-helpers links superseded by the marketplace endpoint + if (/^ai-helpers-(readme|contributing|contributing-skills)$/.test(slug)) { + return highQualityApi.find(apiRecord => + apiRecord.p === 'v6/AI/ai-assisted-development_marketplace/text')?.p; + } + + // 2. Component and development guidelines in ai-helpers + if (doc.path && doc.path.includes('/docs/')) { + const targetSlug = AI_GUIDELINE_ALIASES[slug] || slug; + const candidate = `v6/AI/development-guidelines_${targetSlug}/text`; + const exactMatch = highQualityApi.find(apiRecord => apiRecord.p === candidate); + + if (exactMatch) { + return exactMatch.p; + } + + // Dynamic token fallback against indexed API endpoints + const token = targetSlug.toLowerCase().replace(/[^a-z0-9]/g, ''); + const fuzzyMatch = highQualityApi.find( + apiRecord => + apiRecord.p.startsWith('v6/AI/development-guidelines') && + apiRecord.p.toLowerCase().replace(/[^a-z0-9]/g, '').includes(token) + ); + + return fuzzyMatch?.p; + } + + return undefined; +}; + +/** + * Identify documentation entries in `docs.json` that are redundant with and superseded by the PatternFly API collection. + * + * @param docsCollection - The documentation collection to inspect + * @param [apiRecords=[]] - Records from the API collection seed + * @returns List of redundant records with removal reasons and matched API details + */ +const findApiRedundantDocs = ( + docsCollection: PatternFlyMcpDocsCatalog, + apiRecords: ApiCollectionRecordRef[] = [] +): DocsRemovedRecordReport[] => { + const redundant: DocsRemovedRecordReport[] = []; + const highQualityApi = (apiRecords || []).filter(apiRecord => (apiRecord.q ?? 1) >= 0.95); + + for (const [category, entries] of Object.entries(docsCollection.docs || {})) { + for (const doc of entries) { + // 1. Root uxd-ai-helpers guides must NEVER be pruned (negative control boundary) + if (doc.pathSlug && doc.pathSlug.startsWith('uxd-ai-helpers-')) { + continue; + } + + // 2. Ecosystem / tooling repos must NEVER be pruned + if ( + doc.path && + (doc.path.includes('/patternfly-cli/') || + doc.path.includes('/patternfly-elements/') || + doc.path.includes('/patternfly-mcp/') || + doc.path.includes('/pf-codemods/')) + ) { + continue; + } + + const isAiHelperRepo = + doc.path && + (doc.path.includes('/ai-helpers/') || doc.path.includes('/uxd-ai-helpers/')); + + if (isAiHelperRepo) { + const details = resolveApiEndpointForAiDoc(doc, highQualityApi); + + if (details) { + redundant.push({ + category, + record: doc, + reason: 'superseded by API collection', + details + }); + } + } + } + } + + return redundant; +}; + +/** + * Calculate the diff between an old documentation collection and an updated collection. + * + * @param oldCollection - Original collection + * @param newCollection - Updated collection + * @param [redundantReports=[]] - Explicitly identified removals + * @returns Structured DocsDiffResult + */ +const diffDocsManifests = ( + oldCollection: PatternFlyMcpDocsCatalog, + newCollection: PatternFlyMcpDocsCatalog, + redundantReports: DocsRemovedRecordReport[] = [] +): DocsDiffResult => { + const added: DocsAddedRecordReport[] = []; + const removed: DocsRemovedRecordReport[] = [...redundantReports]; + const modified: DocsModifiedRecordReport[] = []; + const unchanged: DocsUnchangedRecordReport[] = []; + + const oldRecordsMap = new Map(); + + for (const [cat, docs] of Object.entries(oldCollection.docs || {})) { + for (const doc of docs) { + oldRecordsMap.set(`${cat}::${doc.pathSlug}::${doc.displayName}`, { category: cat, doc }); + } + } + + const newRecordsMap = new Map(); + + for (const [cat, docs] of Object.entries(newCollection.docs || {})) { + for (const doc of docs) { + newRecordsMap.set(`${cat}::${doc.pathSlug}::${doc.displayName}`, { category: cat, doc }); + } + } + + const removedKeys = new Set( + redundantReports.map( + report => `${report.category}::${report.record.pathSlug}::${report.record.displayName}` + ) + ); + + // Check for any additional old records missing in new collection + for (const [key, { category, doc }] of oldRecordsMap.entries()) { + if (!newRecordsMap.has(key) && !removedKeys.has(key)) { + removed.push({ + category, + record: doc, + reason: 'removed from manifest', + details: 'Record excluded during manifest update' + }); + removedKeys.add(key); + } + } + + // Check for added, modified, or unchanged in new collection + for (const [key, { category, doc }] of newRecordsMap.entries()) { + const oldEntry = oldRecordsMap.get(key); + + if (!oldEntry) { + added.push({ + category, + record: doc, + reason: 'new upstream document', + details: 'Discovered in manifest update' + }); + continue; + } + + const reasons: string[] = []; + const previousDoc = oldEntry.doc; + const oldHash = extractCommitHash(previousDoc.path || ''); + const newHash = extractCommitHash(doc.path || ''); + + if (previousDoc.path !== doc.path) { + if (oldHash !== newHash && oldHash && newHash) { + const shortOld = oldHash.slice(0, 7); + const shortNew = newHash.slice(0, 7); + + reasons.push(`hash update (${shortOld} -> ${shortNew})`); + } else { + reasons.push('path updated'); + } + } + + if (previousDoc.version !== doc.version) { + reasons.push(`version changed (${previousDoc.version} -> ${doc.version})`); + } + + if (reasons.length > 0) { + modified.push({ + category, + record: doc, + previousRecord: previousDoc, + reasons, + previousHash: oldHash || undefined, + newHash: newHash || undefined + }); + } else { + unchanged.push({ + category, + record: doc + }); + } + } + + return { added, removed, modified, unchanged }; +}; + +/** + * Generate a complete RFC 4180 CSV report for the documentation manifest diff. + * + * @param options - CSV report options or diff result + * @returns Formatted CSV content string + */ +const generateReportCsv = (options: DocsDiffResult | GenerateCsvReportOptions): string => { + const diff = 'diff' in options ? options.diff : options; + + return generateDiffCsv(diff, { + headers: [ + 'status', + 'category', + 'name', + 'pathSlug', + 'path', + 'previousHash', + 'newHash', + 'reason', + 'details' + ], + added: item => [ + item.category, + item.record.displayName, + item.record.pathSlug, + item.record.path, + '', + extractCommitHash(item.record.path) || '', + item.reason || 'new document', + item.details || '' + ], + removed: item => [ + item.category, + item.record.displayName, + item.record.pathSlug, + item.record.path, + extractCommitHash(item.record.path) || '', + '', + item.reason, + item.details || '' + ], + modified: item => [ + item.category, + item.record.displayName, + item.record.pathSlug, + item.record.path, + item.previousHash || '', + item.newHash || '', + item.reasons.join('; '), + item.previousRecord?.path || '' + ], + unchanged: item => { + const hash = extractCommitHash(item.record.path) || ''; + + return [ + item.category, + item.record.displayName, + item.record.pathSlug, + item.record.path, + hash, + hash, + '', + '' + ]; + } + }); +}; + +/** + * Asynchronously recalculate collection entries, prune redundant API records, + * probe reachability for 404 dead links, update hashes, and recompute manifest metadata. + * + * @param collection - Source documentation collection + * @param [options={}] - Options for recalculation + * @returns Promise resolving to a {@link RecalculateManifestResult} containing the + * updated collection and any reachability removals pruned due to broken links + */ +const recalculateManifestMetadata = async ( + collection: PatternFlyMcpDocsCatalog, + options: RecalculateOptions = {} +): Promise => { + const { + redundantRecords = [], + latestHashes, + verifyReachability = false, + probeConcurrency = 15 + } = options; + + const redundantSet = new Set( + redundantRecords.map( + report => `${report.category}::${report.record.pathSlug}::${report.record.displayName}` + ) + ); + + const updatedDocs: PatternFlyMcpDocsCatalogEntry = {}; + const reachabilityRemovals: DocsRemovedRecordReport[] = []; + const allCandidates: Array<{ + category: string; + doc: PatternFlyMcpDocsCatalogDoc; + targetPath: string; + shouldProbe: boolean; + }> = []; + + for (const [category, entries] of Object.entries(collection.docs || {})) { + for (const doc of entries) { + const key = `${category}::${doc.pathSlug}::${doc.displayName}`; + + // 1. Skip records already identified as redundant with the API collection (avoids unnecessary HTTP probes) + if (redundantSet.has(key)) { + continue; + } + + let updatedPath = doc.path; + + // Update commit hash if new hash is available and doc is not a pinned one-off / v5 + if (latestHashes && doc.path && typeof doc.path === 'string') { + const repoInfo = extractRepoInfo(doc.path); + + if (repoInfo && repoInfo.ref !== 'v5') { + const repoKey = `${repoInfo.owner}/${repoInfo.repo}`; + const newSha = latestHashes.get(repoKey) || latestHashes.get(repoInfo.repo); + + // Update if repo is recognized and not a pinned historical reference + if (newSha && !PINNED_HISTORICAL_REFS.has(repoInfo.ref)) { + updatedPath = `https://raw.githubusercontent.com/${repoInfo.owner}/${repoInfo.repo}/${newSha}/${repoInfo.filePath}`; + } + } + } + + allCandidates.push({ + category, + doc, + targetPath: updatedPath, + shouldProbe: verifyReachability && typeof updatedPath === 'string' && updatedPath.startsWith('http') + }); + } + } + + // Global sliding window queue across all categories + const inFlight = new Set>(); + + for (const item of allCandidates) { + const task = (async () => { + let isLive = true; + let targetPath = item.targetPath; + + if (item.shouldProbe) { + isLive = await verifyUrlReachability.memo(item.targetPath); + + // Fallback probe: if newly bumped URL failed, check if previous URL was reachable (path drift detection) + if (!isLive && item.targetPath !== item.doc.path && item.doc.path.startsWith('http')) { + const isPreviousLive = await verifyUrlReachability.memo(item.doc.path); + + if (isPreviousLive) { + targetPath = item.doc.path; + isLive = true; + } + } + } + + if (isLive) { + if (!updatedDocs[item.category]) { + updatedDocs[item.category] = []; + } + + updatedDocs[item.category]?.push({ + ...item.doc, + path: targetPath + }); + } else { + reachabilityRemovals.push({ + category: item.category, + record: item.doc, + reason: 'broken upstream link (HTTP 404)', + details: `Unreachable target: ${item.targetPath}` + }); + } + })().finally(() => { + inFlight.delete(task); + }); + + inFlight.add(task); + if (inFlight.size >= probeConcurrency) { + await Promise.race(inFlight); + } + } + await Promise.all(inFlight); + + const totalEntries = Object.keys(updatedDocs).length; + const totalDocs = Object.values(updatedDocs).reduce((acc, arr) => acc + arr.length, 0); + + return { + collection: { + version: collection.version || '1', + generated: new Date().toISOString(), + meta: { + totalEntries, + totalDocs, + source: collection.meta?.source || 'patternfly-mcp' + }, + docs: updatedDocs + }, + reachabilityRemovals + }; +}; + +export { + AI_GUIDELINE_ALIASES, + PINNED_HISTORICAL_REFS, + diffDocsManifests, + findApiRedundantDocs, + generateReportCsv, + recalculateManifestMetadata, + resolveApiEndpointForAiDoc, + type ApiCollectionRecordRef, + type GenerateCsvReportOptions, + type DocsAddedRecordReport, + type DocsDiffResult, + type DocsModifiedRecordReport, + type DocsRemovedRecordReport, + type DocsUnchangedRecordReport, + type RecalculateManifestResult, + type RecalculateOptions +}; From 3c9bf570529606869d0deb7707fc1a5a77130100 Mon Sep 17 00:00:00 2001 From: CD Cabrera Date: Wed, 21 Oct 2026 13:34:38 -0400 Subject: [PATCH 2/5] fix: review update --- docs/development.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/development.md b/docs/development.md index b61e704b..d13e491a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -546,7 +546,7 @@ The server maintains multiple record collections that provide documentation and When PatternFly releases new documentation or component features, reseed the embedded local collection data: - `npm run build:collections`: Runs reseeding for all collections and collection tests. - `npm run build:collections-api`: Updates `src/collection.patternFlyApi.json` and outputs CSV diffs to `reports/`. -- `npm run build:collections-docs`: Updates `src/docs.json` with latest repository hashes and prunes API duplicates. +- `npm run build:collections-docs`: Updates `src/docs.json` with latest repository hashes. #### Unique collections @@ -567,7 +567,7 @@ The original curated Markdown collection. It pins remote repository URLs to spec # Run collection test suites in watch mode npm run test:collections-dev ``` -- **Automated synchronization**: The Docs reseeding pipeline (`tooling/collections/update.patternFlyDocs.ts`) automatically resolves the latest commit SHAs across referenced GitHub repositories, prunes redundant AI guideline entries superseded by high-quality API endpoints, recalculates `meta.totalEntries` and `meta.totalDocs`, and generates an audit diff report at `reports/YYYYMMDD-collection.patternFlyDocs.report.csv`. +- **Automated synchronization**: The Docs reseeding pipeline (`tooling/collections/update.patternFlyDocs.ts`) automatically resolves the latest commit SHAs across referenced GitHub repositories, recalculates `meta.totalEntries` and `meta.totalDocs`, and generates an audit diff report at `reports/YYYYMMDD-collection.patternFlyDocs.report.csv`. - **Validation**: Ensure all raw URLs are HTTPS, match the domain whitelist in `src/options.defaults.ts`, and return 2xx responses. - **Testing**: Run `npm run test:collections` or `npm test`, updating `baseHashes` and the Repository Breakdown table in `src/__tests__/docs.json.test.ts` when refs change. From b1f1c99bf5eea8c623e59ef4899eae765ab05ee2 Mon Sep 17 00:00:00 2001 From: CD Cabrera Date: Wed, 21 Oct 2026 13:53:35 -0400 Subject: [PATCH 3/5] fix: review update --- tooling/collections/__tests__/helpers.test.ts | 16 ++++++++-------- tooling/collections/helpers.ts | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tooling/collections/__tests__/helpers.test.ts b/tooling/collections/__tests__/helpers.test.ts index eb4d6b03..1f6137f1 100644 --- a/tooling/collections/__tests__/helpers.test.ts +++ b/tooling/collections/__tests__/helpers.test.ts @@ -21,7 +21,7 @@ jest.mock('node:fs/promises', () => ({ jest.mock('node:child_process', () => ({ ...jest.requireActual('node:child_process'), - execSync: jest.fn() + execFileSync: jest.fn() })); const mockWriteFile = writeFile as jest.MockedFunction; @@ -224,7 +224,7 @@ describe('fetchCommitViaGitLsRemote', () => { }); it('should return commit SHA when git ls-remote output contains SHA with HEAD', () => { - jest.spyOn(childProcess, 'execSync').mockReturnValue( + jest.spyOn(childProcess, 'execFileSync').mockReturnValue( 'ref: refs/heads/main\tHEAD\n8f4a382e783457a4128509789234857234895723\tHEAD\n' ); @@ -234,7 +234,7 @@ describe('fetchCommitViaGitLsRemote', () => { }); it('should fallback to 40-character SHA if symref line is missing', () => { - jest.spyOn(childProcess, 'execSync').mockReturnValue( + jest.spyOn(childProcess, 'execFileSync').mockReturnValue( '8f4a382e783457a4128509789234857234895723\trefs/heads/main\n' ); @@ -244,7 +244,7 @@ describe('fetchCommitViaGitLsRemote', () => { }); it('should return undefined if command fails or throws', () => { - jest.spyOn(childProcess, 'execSync').mockImplementation(() => { + jest.spyOn(childProcess, 'execFileSync').mockImplementation(() => { throw new Error('Command failed'); }); @@ -291,7 +291,7 @@ describe('fetchRepoCommit', () => { it('should fallback to git ls-remote when GitHub API fails', async () => { jest.spyOn(global, 'fetch').mockRejectedValue(new Error('Network error')); - jest.spyOn(childProcess, 'execSync').mockReturnValue( + jest.spyOn(childProcess, 'execFileSync').mockReturnValue( '0123456789abcdef0123456789abcdef01234567\tHEAD\n' ); @@ -302,7 +302,7 @@ describe('fetchRepoCommit', () => { it('should return undefined when API fails and git ls-remote fails', async () => { jest.spyOn(global, 'fetch').mockRejectedValue(new Error('Network error')); - jest.spyOn(childProcess, 'execSync').mockImplementation(() => { + jest.spyOn(childProcess, 'execFileSync').mockImplementation(() => { throw new Error('git error'); }); @@ -410,7 +410,7 @@ describe('fetchLatestRepoHashes', () => { it('should fall back to git ls-remote when GitHub API fails', async () => { jest.spyOn(global, 'fetch').mockRejectedValue(new Error('Rate limit exceeded')); - jest.spyOn(childProcess, 'execSync').mockReturnValue( + jest.spyOn(childProcess, 'execFileSync').mockReturnValue( '0123456789abcdef0123456789abcdef01234567\tHEAD\n' ); @@ -423,7 +423,7 @@ describe('fetchLatestRepoHashes', () => { it('should handle fetch failures gracefully without throwing', async () => { jest.spyOn(global, 'fetch').mockRejectedValue(new Error('Rate limit exceeded')); - jest.spyOn(childProcess, 'execSync').mockImplementation(() => { + jest.spyOn(childProcess, 'execFileSync').mockImplementation(() => { throw new Error('git command failed'); }); diff --git a/tooling/collections/helpers.ts b/tooling/collections/helpers.ts index 7d9af90d..9f6efbbb 100644 --- a/tooling/collections/helpers.ts +++ b/tooling/collections/helpers.ts @@ -1,4 +1,4 @@ -import { execSync } from 'node:child_process'; +import { execFileSync } from 'node:child_process'; import { writeFile } from 'node:fs/promises'; import { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -117,7 +117,7 @@ const extractTrackedReposFromCollection = ( const fetchCommitViaGitLsRemote = (owner: string, repo: string): string | undefined => { try { const remoteUrl = `https://github.com/${owner}/${repo}.git`; - const output = execSync(`git ls-remote --symref ${remoteUrl} HEAD`, { + const output = execFileSync('git', ['ls-remote', '--symref', remoteUrl, 'HEAD'], { encoding: 'utf-8', timeout: 10_000, stdio: ['ignore', 'pipe', 'ignore'] From 08026eb52ec0fbd52451f84eabd6887bd83ce682 Mon Sep 17 00:00:00 2001 From: CD Cabrera Date: Wed, 21 Oct 2026 14:02:31 -0400 Subject: [PATCH 4/5] fix: review update --- tooling/collections/update.patternFlyDocs.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tooling/collections/update.patternFlyDocs.ts b/tooling/collections/update.patternFlyDocs.ts index ab9db3cd..e175bcfd 100644 --- a/tooling/collections/update.patternFlyDocs.ts +++ b/tooling/collections/update.patternFlyDocs.ts @@ -71,6 +71,11 @@ const logDiffReport = (diff: DocsDiffResult) => { /** * Update the PatternFly Docs manifest collection, deduplicating against the API seed and syncing repository SHAs. * + * @note `verifyReachability` is currently defaulted to `false` because reachability checks are destructive. + * Records failing reachability probes are actively dropped from `docs.json` when the older path may still be valid. + * This setting was meant to compliment the deduplication of records found within the PF API collection. + * Once stability issues have been addressed within the PF API service, we'll review defaulting this to `true`. + * * @param [options] - Optional execution options. * @param [options.docsFilename=DEFAULT_DOCS_FILENAME] - Target documentation filename. * @param [options.apiFilename=DEFAULT_API_FILENAME] - Target API filename for deduplication. From 183667f67fd7a005cb6f8693e3fb83bf96bfd3d6 Mon Sep 17 00:00:00 2001 From: CD Cabrera Date: Wed, 21 Oct 2026 15:29:42 -0400 Subject: [PATCH 5/5] fix: review update --- tooling/collections/__tests__/helpers.test.ts | 52 +++++++++++++------ tooling/collections/helpers.ts | 7 +-- 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/tooling/collections/__tests__/helpers.test.ts b/tooling/collections/__tests__/helpers.test.ts index 1f6137f1..7c0f444f 100644 --- a/tooling/collections/__tests__/helpers.test.ts +++ b/tooling/collections/__tests__/helpers.test.ts @@ -25,11 +25,13 @@ jest.mock('node:child_process', () => ({ })); const mockWriteFile = writeFile as jest.MockedFunction; +const mockExecFileSync = childProcess.execFileSync as jest.MockedFunction; afterEach(() => { extractRepoInfo.memo.clear(); fetchRepoCommit.memo.clear(); verifyUrlReachability.memo.clear(); + mockExecFileSync.mockReset(); }); describe('extractTrackedReposFromCollection', () => { @@ -223,24 +225,23 @@ describe('fetchCommitViaGitLsRemote', () => { jest.restoreAllMocks(); }); - it('should return commit SHA when git ls-remote output contains SHA with HEAD', () => { - jest.spyOn(childProcess, 'execFileSync').mockReturnValue( - 'ref: refs/heads/main\tHEAD\n8f4a382e783457a4128509789234857234895723\tHEAD\n' - ); - - const sha = fetchCommitViaGitLsRemote('patternfly', 'patternfly-react'); - - expect(sha).toBe('8f4a382e783457a4128509789234857234895723'); - }); - - it('should fallback to 40-character SHA if symref line is missing', () => { - jest.spyOn(childProcess, 'execFileSync').mockReturnValue( - '8f4a382e783457a4128509789234857234895723\trefs/heads/main\n' - ); + it.each([ + { + description: 'git ls-remote output contains SHA with HEAD', + value: 'ref: refs/heads/main\tHEAD\n8f4a382e783457a4128509789234857234895723\tHEAD\n', + expected: '8f4a382e783457a4128509789234857234895723' + }, + { + description: 'fallback to 40-character SHA if symref line is missing', + value: '8f4a382e783457a4128509789234857234895723\trefs/heads/main\n', + expected: '8f4a382e783457a4128509789234857234895723' + } + ])('should return commit SHA, $description', ({ value, expected }) => { + jest.spyOn(childProcess, 'execFileSync').mockReturnValue(value); const sha = fetchCommitViaGitLsRemote('patternfly', 'patternfly-react'); - expect(sha).toBe('8f4a382e783457a4128509789234857234895723'); + expect(sha).toBe(expected); }); it('should return undefined if command fails or throws', () => { @@ -252,6 +253,16 @@ describe('fetchCommitViaGitLsRemote', () => { expect(sha).toBeUndefined(); }); + + it('should query specific ref when provided', () => { + const spy = jest.spyOn(childProcess, 'execFileSync').mockReturnValue( + '8f4a382e783457a4128509789234857234895723\trefs/heads/v5\n' + ); + const sha = fetchCommitViaGitLsRemote('patternfly', 'patternfly-react', 'v5'); + + expect(sha).toBe('8f4a382e783457a4128509789234857234895723'); + expect(spy).toHaveBeenCalledWith('git', ['ls-remote', '--symref', 'https://github.com/patternfly/patternfly-react.git', 'v5'], expect.any(Object)); + }); }); describe('fetchRepoCommit', () => { @@ -326,6 +337,17 @@ describe('fetchRepoCommit', () => { expect(second).toBe('mock-memo-sha'); expect(fetchSpy).toHaveBeenCalledTimes(1); }); + + it('should fallback to git ls-remote with the requested branch when API fails', async () => { + jest.spyOn(global, 'fetch').mockRejectedValue(new Error('Rate limit exceeded')); + const spy = jest.spyOn(childProcess, 'execFileSync').mockReturnValue( + 'fedcba9876543210fedcba9876543210fedcba98\trefs/heads/v5\n' + ); + const sha = await fetchRepoCommit('patternfly', 'patternfly-react', 'v5'); + + expect(sha).toBe('fedcba9876543210fedcba9876543210fedcba98'); + expect(spy).toHaveBeenCalledWith('git', ['ls-remote', '--symref', 'https://github.com/patternfly/patternfly-react.git', 'v5'], expect.any(Object)); + }); }); describe('fetchLatestRepoHashes', () => { diff --git a/tooling/collections/helpers.ts b/tooling/collections/helpers.ts index 9f6efbbb..ccee84d5 100644 --- a/tooling/collections/helpers.ts +++ b/tooling/collections/helpers.ts @@ -112,12 +112,13 @@ const extractTrackedReposFromCollection = ( * * @param owner - Repository owner/organization * @param repo - Repository name + * @param [ref='HEAD'] - Branch, tag, or commit ref to resolve * @returns Commit SHA or undefined if git is unavailable or command fails */ -const fetchCommitViaGitLsRemote = (owner: string, repo: string): string | undefined => { +const fetchCommitViaGitLsRemote = (owner: string, repo: string, ref = 'HEAD'): string | undefined => { try { const remoteUrl = `https://github.com/${owner}/${repo}.git`; - const output = execFileSync('git', ['ls-remote', '--symref', remoteUrl, 'HEAD'], { + const output = execFileSync('git', ['ls-remote', '--symref', remoteUrl, ref], { encoding: 'utf-8', timeout: 10_000, stdio: ['ignore', 'pipe', 'ignore'] @@ -238,7 +239,7 @@ const fetchRepoCommit = async ( // Fallback on network/API failure } - return sha || fetchCommitViaGitLsRemote(owner, repo); + return sha || fetchCommitViaGitLsRemote(owner, repo, branch); }; /**