diff --git a/src/utils/docs.functions.ts b/src/utils/docs.functions.ts index 71969bf16..5c53fbb5e 100644 --- a/src/utils/docs.functions.ts +++ b/src/utils/docs.functions.ts @@ -1,7 +1,6 @@ import { notFound } from '@tanstack/react-router' import { createServerFn, createServerOnlyFn } from '@tanstack/react-start' import { setResponseHeader } from '@tanstack/react-start/server' -import removeMarkdown from 'remove-markdown' import * as v from 'valibot' import { extractFrameworksFromMarkdown } from './markdown/filterFrameworkContent' import { buildRedirectManifest, type RedirectManifestEntry } from './redirects' @@ -437,8 +436,7 @@ export const fetchDocs = createServerFn({ method: 'GET' }) const { extractFrontMatter } = await loadDocumentsServerModule() const frontMatter = extractFrontMatter(result.file) - const description = - frontMatter.userDescription ?? removeMarkdown(frontMatter.excerpt ?? '') + const description = frontMatter.data.description const keywords = extractFrontMatterKeywords(frontMatter.data.keywords) setDocsCacheHeaders( diff --git a/src/utils/documents.server.ts b/src/utils/documents.server.ts index 656995c4d..d7828d009 100644 --- a/src/utils/documents.server.ts +++ b/src/utils/documents.server.ts @@ -3,6 +3,8 @@ import fsp from 'node:fs/promises' import os from 'node:os' import path from 'node:path' import { parse as parseYaml } from 'yaml' +import { parseFragment } from 'parse5' +import type { BlockNode, InlineNode } from '@tanstack/markdown' import { getCurrentHostRuntimeEnv, getHostRuntimeEnv, @@ -24,6 +26,7 @@ import { isValidRepoPath } from './repo-path' import { multiSortBy, removeLeadingSlash } from './utils' import { env } from './env' import { fetchWithTimeout } from './outbound-fetch.server' +import { parseSiteMarkdown } from './markdown/processor' type FrontMatterValue = | string @@ -842,9 +845,10 @@ export function extractFrontMatter(content: string) { const title = typeof parsed.data.title === 'string' ? parsed.data.title : undefined const ref = typeof parsed.data.ref === 'string' ? parsed.data.ref : undefined + const excerpt = createExcerpt(parsed.content) const data: FrontMatterData = { ...parsed.data, - description: userDescription ?? createExcerpt(parsed.content), + description: userDescription ?? excerpt, title, ref, redirect_from: redirectFrom, @@ -854,7 +858,7 @@ export function extractFrontMatter(content: string) { return { content: parsed.content, data, - excerpt: createRichExcerpt(parsed.content), + excerpt: createRichExcerpt(parsed.content, excerpt), userDescription, } } @@ -935,28 +939,65 @@ function toFrontMatterValue(value: unknown): FrontMatterValue | undefined { } function createExcerpt(text: string, maxLength = 200) { - // Remove Markdown formatting using a basic regex - - let cleanText = text - .replace(/!\[.*?\]\(.*?\)/g, '') // Remove images - .replace(/\[.*?\]\(.*?\)/g, '') // Remove links - .replace(/[`*_~>]/g, '') // Remove Markdown special characters - .replace(/#+\s/g, '') // Remove headers - .replace(/-\s/g, '') // Remove list markers - .replace(/\r?\n|\r/g, ' ') // Convert line breaks to spaces - .replace(/\s+/g, ' ') // Collapse multiple spaces + const cleanText = excerptBlocks(parseSiteMarkdown(text).children) + .replace(/\s+/g, ' ') .trim() - // Truncate the text to the desired length, preserving whole words if (cleanText.length > maxLength) { - cleanText = cleanText.slice(0, maxLength).trim() + '...' + const end = cleanText.lastIndexOf(' ', maxLength) + return cleanText.slice(0, end > 0 ? end : maxLength) + '...' } return cleanText } -function createRichExcerpt(text: string, maxLength = 200) { - let cleanText = createExcerpt(text, maxLength) +function excerptBlocks(blocks: Array): string { + return blocks + .map((block): string => { + switch (block.type) { + case 'paragraph': + return excerptInline(block.children) + case 'blockquote': + case 'callout': + case 'component': + return excerptBlocks(block.children) + case 'list': + return block.items + .map((item) => excerptBlocks(item.children)) + .join(' ') + default: + return '' + } + }) + .join(' ') +} + +function excerptInline(nodes: Array): string { + return nodes + .map((node): string => { + switch (node.type) { + case 'text': + return parseFragment(node.value.replaceAll('<', '<')) + .childNodes.map((child) => ('value' in child ? child.value : '')) + .join('') + case 'inlineCode': + return node.value + case 'link': + case 'strong': + case 'emphasis': + case 'strike': + return excerptInline(node.children) + case 'break': + return ' ' + default: + return '' + } + }) + .join('') +} + +function createRichExcerpt(text: string, excerpt: string) { + let cleanText = excerpt const imageText = extractFirstImage(text) diff --git a/tests/document-descriptions.test.ts b/tests/document-descriptions.test.ts new file mode 100644 index 000000000..ab777203c --- /dev/null +++ b/tests/document-descriptions.test.ts @@ -0,0 +1,72 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { extractFrontMatter } from '../src/utils/documents.server' + +test('descriptions preserve linked prose without headings, images, or callout markers', () => { + const result = extractFrontMatter(`--- +title: Overview +--- +# Overview + +![Start logo](https://example.com/logo.png) + +> [!NOTE] +> Build with [**TanStack Router**](https://tanstack.com/router) and \`createServerFn\`. + +Use server-side rendering & typed routes. + +\`\`\`tsx +const implementationDetails = true +\`\`\` +`) + + assert.equal( + result.data.description, + 'Build with TanStack Router and createServerFn. Use server-side rendering & typed routes.', + ) + assert.match(result.excerpt, /^!\[Start logo\]/) + assert.doesNotMatch( + result.data.description, + /logo|height:|implementationDetails/, + ) +}) + +test('descriptions preserve reference links, inline identifiers, and line breaks', () => { + const result = + extractFrontMatter(`Use [TanStack Query][query] with \`get_current_user\`. + +Line one +Line two. + +[query]: https://tanstack.com/query +`) + assert.equal( + result.data.description, + 'Use TanStack Query with get_current_user. Line one Line two.', + ) +}) + +test('authored descriptions take precedence over generated excerpts', () => { + const result = extractFrontMatter(`--- +description: Learn how to deploy TanStack Start. +--- +Unrelated opening paragraph. +`) + assert.equal(result.data.description, 'Learn how to deploy TanStack Start.') +}) + +test('entity decoding preserves code type parameters and literal entity examples', () => { + const result = extractFrontMatter( + 'Use `Array` with "quoted" values, 🚀, and literal `&`.', + ) + assert.equal( + result.data.description, + 'Use Array with "quoted" values, 🚀, and literal &.', + ) +}) + +test('generated descriptions end on a word boundary', () => { + const result = extractFrontMatter('Short words. '.repeat(30)) + assert.ok(result.data.description.length <= 203) + assert.match(result.data.description, /(?:Short|words\.)\.\.\.$/) +})