Fix generated documentation descriptions

src/utils/docs.functions.ts

@@ -1,7 +1,6 @@
 import { notFound } from '@tanstack/react-router'
 import { createServerFn, createServerOnlyFn } from '@tanstack/react-start'
 import { setResponseHeader } from '@tanstack/react-start/server'
-import removeMarkdown from 'remove-markdown'
 import * as v from 'valibot'
 import { extractFrameworksFromMarkdown } from './markdown/filterFrameworkContent'
 import { buildRedirectManifest, type RedirectManifestEntry } from './redirects'
@@ -437,8 +436,7 @@ export const fetchDocs = createServerFn({ method: 'GET' })
 
     const { extractFrontMatter } = await loadDocumentsServerModule()
     const frontMatter = extractFrontMatter(result.file)
-    const description =
-      frontMatter.userDescription ?? removeMarkdown(frontMatter.excerpt ?? '')
+    const description = frontMatter.data.description
     const keywords = extractFrontMatterKeywords(frontMatter.data.keywords)
 
     setDocsCacheHeaders(

src/utils/documents.server.ts

@@ -3,6 +3,8 @@ import fsp from 'node:fs/promises'
 import os from 'node:os'
 import path from 'node:path'
 import { parse as parseYaml } from 'yaml'
+import { parseFragment } from 'parse5'
+import type { BlockNode, InlineNode } from '@tanstack/markdown'
 import {
   getCurrentHostRuntimeEnv,
   getHostRuntimeEnv,
@@ -24,6 +26,7 @@ import { isValidRepoPath } from './repo-path'
 import { multiSortBy, removeLeadingSlash } from './utils'
 import { env } from './env'
 import { fetchWithTimeout } from './outbound-fetch.server'
+import { parseSiteMarkdown } from './markdown/processor'
 
 type FrontMatterValue =
   | string
@@ -842,9 +845,10 @@ export function extractFrontMatter(content: string) {
   const title =
     typeof parsed.data.title === 'string' ? parsed.data.title : undefined
   const ref = typeof parsed.data.ref === 'string' ? parsed.data.ref : undefined
+  const excerpt = createExcerpt(parsed.content)
   const data: FrontMatterData = {
     ...parsed.data,
-    description: userDescription ?? createExcerpt(parsed.content),
+    description: userDescription ?? excerpt,
     title,
     ref,
     redirect_from: redirectFrom,
@@ -854,7 +858,7 @@ export function extractFrontMatter(content: string) {
   return {
     content: parsed.content,
     data,
-    excerpt: createRichExcerpt(parsed.content),
+    excerpt: createRichExcerpt(parsed.content, excerpt),
     userDescription,
   }
 }
@@ -935,28 +939,65 @@ function toFrontMatterValue(value: unknown): FrontMatterValue | undefined {
 }
 
 function createExcerpt(text: string, maxLength = 200) {
-  // Remove Markdown formatting using a basic regex
-
-  let cleanText = text
-    .replace(/!\[.*?\]\(.*?\)/g, '') // Remove images
-    .replace(/\[.*?\]\(.*?\)/g, '') // Remove links
-    .replace(/[`*_~>]/g, '') // Remove Markdown special characters
-    .replace(/#+\s/g, '') // Remove headers
-    .replace(/-\s/g, '') // Remove list markers
-    .replace(/\r?\n|\r/g, ' ') // Convert line breaks to spaces
-    .replace(/\s+/g, ' ') // Collapse multiple spaces
+  const cleanText = excerptBlocks(parseSiteMarkdown(text).children)
+    .replace(/\s+/g, ' ')
     .trim()
 
-  // Truncate the text to the desired length, preserving whole words
   if (cleanText.length > maxLength) {
-    cleanText = cleanText.slice(0, maxLength).trim() + '...'
+    const end = cleanText.lastIndexOf(' ', maxLength)
+    return cleanText.slice(0, end > 0 ? end : maxLength) + '...'
   }
 
   return cleanText
 }
 
-function createRichExcerpt(text: string, maxLength = 200) {
-  let cleanText = createExcerpt(text, maxLength)
+function excerptBlocks(blocks: Array<BlockNode>): string {
+  return blocks
+    .map((block): string => {
+      switch (block.type) {
+        case 'paragraph':
+          return excerptInline(block.children)
+        case 'blockquote':
+        case 'callout':
+        case 'component':
+          return excerptBlocks(block.children)
+        case 'list':
+          return block.items
+            .map((item) => excerptBlocks(item.children))
+            .join(' ')
+        default:
+          return ''
+      }
+    })
+    .join(' ')
+}
+
+function excerptInline(nodes: Array<InlineNode>): string {
+  return nodes
+    .map((node): string => {
+      switch (node.type) {
+        case 'text':
+          return parseFragment(node.value.replaceAll('<', '&lt;'))
+            .childNodes.map((child) => ('value' in child ? child.value : ''))
+            .join('')
+        case 'inlineCode':
+          return node.value
+        case 'link':
+        case 'strong':
+        case 'emphasis':
+        case 'strike':
+          return excerptInline(node.children)
+        case 'break':
+          return ' '
+        default:
+          return ''
+      }
+    })
+    .join('')
+}
+
+function createRichExcerpt(text: string, excerpt: string) {
+  let cleanText = excerpt
 
   const imageText = extractFirstImage(text)
 

tests/document-descriptions.test.ts

@@ -0,0 +1,72 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { extractFrontMatter } from '../src/utils/documents.server'
+
+test('descriptions preserve linked prose without headings, images, or callout markers', () => {
+  const result = extractFrontMatter(`---
+title: Overview
+---
+# Overview
+
+![Start logo](https://example.com/logo.png)
+
+> [!NOTE]
+> Build with [**TanStack Router**](https://tanstack.com/router) and \`createServerFn\`.
+
+Use <strong>server-side</strong> rendering &amp; typed routes.
+
+\`\`\`tsx
+const implementationDetails = true
+\`\`\`
+`)
+
+  assert.equal(
+    result.data.description,
+    'Build with TanStack Router and createServerFn. Use server-side rendering & typed routes.',
+  )
+  assert.match(result.excerpt, /^!\[Start logo\]/)
+  assert.doesNotMatch(
+    result.data.description,
+    /logo|height:|implementationDetails/,
+  )
+})
+
+test('descriptions preserve reference links, inline identifiers, and line breaks', () => {
+  const result =
+    extractFrontMatter(`Use [TanStack Query][query] with \`get_current_user\`.
+
+Line one  
+Line two.
+
+[query]: https://tanstack.com/query
+`)
+  assert.equal(
+    result.data.description,
+    'Use TanStack Query with get_current_user. Line one Line two.',
+  )
+})
+
+test('authored descriptions take precedence over generated excerpts', () => {
+  const result = extractFrontMatter(`---
+description: Learn how to deploy TanStack Start.
+---
+Unrelated opening paragraph.
+`)
+  assert.equal(result.data.description, 'Learn how to deploy TanStack Start.')
+})
+
+test('entity decoding preserves code type parameters and literal entity examples', () => {
+  const result = extractFrontMatter(
+    'Use `Array<T>` with &quot;quoted&quot; values, &#x1F680;, and literal `&amp;`.',
+  )
+  assert.equal(
+    result.data.description,
+    'Use Array<T> with "quoted" values, 🚀, and literal &amp;.',
+  )
+})
+
+test('generated descriptions end on a word boundary', () => {
+  const result = extractFrontMatter('Short words. '.repeat(30))
+  assert.ok(result.data.description.length <= 203)
+  assert.match(result.data.description, /(?:Short|words\.)\.\.\.$/)
+})