fix(docs): preserve retrieval excerpts and reduce reading overhead

Author: emir-karabeg

.github/workflows/docs-negotiation.yml

@@ -96,6 +96,7 @@ jobs:
           echo "DOCS_BENCHMARK_REVISION=$REVISION" >> "$GITHUB_ENV"
           bunx --no-install playwright install --with-deps chromium
       - name: Build measured production docs
+        id: build
         if: matrix.target == 'build'
         run: bun run --cwd apps/docs build
       - name: Measure navigation and Lighthouse sequentially
@@ -128,6 +129,16 @@ jobs:
           name: docs-benchmark-${{ matrix.target }}
           path: test-results/benchmark/
           retention-days: 14
+      - name: Package measured production preview
+        if: always() && inputs.preview_artifact && steps.build.outcome == 'success'
+        run: tar -czf docs-preview-build.tar.gz --exclude='.next/cache' -C apps/docs .next .content
+      - name: Upload measured production preview
+        if: always() && inputs.preview_artifact && steps.build.outcome == 'success'
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: docs-preview-build
+          path: docs-preview-build.tar.gz
+          retention-days: 3
 
   http:
     if: github.event_name != 'workflow_dispatch' || !inputs.benchmark

apps/docs/ARCHITECTURE.md

@@ -67,6 +67,6 @@ The implementation was checked against the installed Next.js 16.4.1 docs and cur
 
 Review the migration report first, then the library gallery and representative pages, then the HTTP/browser reports. Run a production build without embedding credentials to verify the local fallback. Production search index publication remains a separate deployment step and must pass its freshness check.
 
-This change preserves the docs workspace, origin, deployment identity, product generators, and content paths. Reverting the docs replacement commit and its lockfile/workflow changes restores the prior implementation; it requires no database migration or product-data rollback. Keep the PR in draft until the visual and behavior evidence has been reviewed.
+This change preserves the docs workspace, origin, deployment identity, product generators, and content paths. Restoring the previous docs deployment requires no schema migration or product-data rollback. Search publication changes `docs_embeddings` separately: before publishing, back up that table and record its paired docs deployment and source revision. Rollback must hold queued publishers, restore the paired deployment and index, and check freshness and citations from the restored source. The workflow does not create this backup automatically and checks out current `main`; ordinary dispatch cannot select an older publisher. Keep the PR in draft until the visual and behavior evidence has been reviewed.
 
 The requested move to `https://sim.ai/docs` requires a coordinated path-aware build and infrastructure cutover. See [the domain migration plan](migration/domain-move.md); the current build remains on `docs.sim.ai` until that work is verified.

apps/docs/MACHINE_ACCESS.md

@@ -47,6 +47,8 @@ All three tools advertise read-only behavior. Unknown pages produce tool errors;
 
 Basic search uses a server-only index generated atomically with the local compiled catalog and needs no database or embedding credentials. Markdown parsing and text normalization happen during content compilation; requests read precomputed chunks. The generated index is capped at 64 MiB and its catalog fingerprint is validated before serving results. When `DATABASE_URL` is present, bounded PostgreSQL keyword search can supplement it; `OPENAI_API_KEY` additionally enables vector search using the existing 1536-dimensional `text-embedding-3-small` table contract. Unavailable providers fall back to lexical results. Results with old content hashes or nonexistent heading targets are excluded.
 
+Search chunks contain at most 6,000 characters, including continuation context. Paragraphs and code blocks stay intact when they fit. Larger code blocks become explicitly labeled excerpts with the original language, complete fences and unchanged code lines; an excerpt is not necessarily a complete JSON value or executable program. Continuations repeat the page and heading, and table excerpts repeat their column headers while retaining oversized cell text in fragments. Content that cannot fit safely, such as one oversized code line or an oversized nested code block, stops compilation with the page and heading in the error instead of discarding content.
+
 From `apps/docs`:
 
 ```sh

apps/docs/components/ui/scroll-region.tsx

@@ -19,7 +19,6 @@ export function ScrollRegion({ label, className, children, ...props }: ScrollReg
     const observer = new ResizeObserver(updateFocusability)
     observer.observe(element)
     if (element.firstElementChild) observer.observe(element.firstElementChild)
-    updateFocusability()
     return () => observer.disconnect()
   }, [])
   return (

apps/docs/hooks/queries/search.ts

@@ -1,21 +1,21 @@
 'use client'
 
 import { useQuery } from '@tanstack/react-query'
-import { array, literal, object, string } from 'zod/mini'
+import { z } from 'zod'
 
 const SEARCH_STALE_TIME = 60_000
 const SEARCH_GC_TIME = 300_000
 const searchKeys = {
   all: ['docs-search'] as const,
   query: (query: string) => [...searchKeys.all, query] as const,
 }
-const resultsSchema = array(
-  object({
-    id: string(),
-    type: literal('page'),
-    url: string(),
-    content: string(),
-    breadcrumbs: array(string()),
+const resultsSchema = z.array(
+  z.object({
+    id: z.string(),
+    type: z.literal('page'),
+    url: z.string(),
+    content: z.string(),
+    breadcrumbs: z.array(z.string()),
   })
 )
 

apps/docs/lib/search/chunks.ts

@@ -1,3 +1,4 @@
+import type { Code, RootContent } from 'mdast'
 import { toString as markdownToString } from 'mdast-util-to-string'
 import remarkParse from 'remark-parse'
 import { unified } from 'unified'
@@ -9,6 +10,133 @@ import { DOCS_BASE_URL } from '@/lib/urls'
 const CHUNK_CHARACTERS = 6000
 const parser = unified().use(remarkParse)
 
+function splitText(value: string, limit: number, location: string): string[] {
+  const parts: string[] = []
+  let rest = value
+  while (rest.length > limit) {
+    const boundary = Math.max(rest.lastIndexOf(' ', limit), rest.lastIndexOf('\n', limit))
+    if (boundary <= 0) throw new Error(`${location}: unbreakable text exceeds search chunk budget`)
+    parts.push(rest.slice(0, boundary).trimEnd())
+    rest = rest.slice(boundary).trimStart()
+  }
+  if (rest || !parts.length) parts.push(rest)
+  return parts
+}
+
+function packLines(lines: string[], limit: number, location: string): string[] {
+  const parts: string[] = []
+  let current: string[] = []
+  let length = 0
+  for (const line of lines) {
+    if (line.length > limit) throw new Error(`${location}: a code line exceeds search chunk budget`)
+    if (current.length && length + 1 + line.length > limit) {
+      parts.push(current.join('\n'))
+      current = []
+      length = 0
+    }
+    length += line.length + (current.length ? 1 : 0)
+    current.push(line)
+  }
+  if (current.length) parts.push(current.join('\n'))
+  return parts
+}
+
+function excerptLabel(kind: string, index: number, total: number) {
+  return `${kind} excerpt (part ${index} of ${total}; continued).\n\n`
+}
+
+function codeParts(node: Code, raw: string, limit: number, location: string) {
+  const opening = /^ {0,3}(`{3,}|~{3,})([^\n]*)/.exec(raw)
+  const marker = opening?.[1] ?? '```'
+  const opener = opening ? `${marker}${opening[2]}` : marker
+  const labelBudget = excerptLabel('Code', raw.length, raw.length).length
+  const parts = packLines(
+    node.value.split('\n'),
+    limit - opener.length - marker.length - labelBudget - 2,
+    location
+  )
+  return parts.map(
+    (part, index) => `${excerptLabel('Code', index + 1, parts.length)}${opener}\n${part}\n${marker}`
+  )
+}
+
+function tableCells(row: string) {
+  const cells: string[] = []
+  let current = ''
+  let escapes = 0
+  for (const character of row.trim().replace(/^\|/, '').replace(/\|$/, '')) {
+    if (character === '|' && escapes % 2 === 0) {
+      cells.push(current.trim())
+      current = ''
+    } else current += character
+    escapes = character === '\\' ? escapes + 1 : 0
+  }
+  cells.push(current.trim())
+  return cells
+}
+
+function tableParts(raw: string, limit: number, location: string) {
+  const [header, separator, ...rows] = raw.split('\n')
+  const prefix = `${header}\n${separator}\n`
+  const rowBudget = limit - prefix.length - excerptLabel('Table', raw.length, raw.length).length
+  if (rowBudget < 100) throw new Error(`${location}: table header exceeds search chunk budget`)
+  const expanded = rows.flatMap((row) => {
+    if (row.length <= rowBudget) return [row]
+    const cells = tableCells(row)
+    const cellBudget = Math.floor((rowBudget - 3 * cells.length - 1) / cells.length)
+    if (cellBudget < 1) throw new Error(`${location}: table columns exceed search chunk budget`)
+    const fragments = cells.map((cell) => splitText(cell, cellBudget, location))
+    return Array.from(
+      { length: Math.max(...fragments.map((parts) => parts.length)) },
+      (_, index) =>
+        `| ${fragments.map((parts) => parts[index] ?? (parts.length === 1 ? parts[0] : '')).join(' | ')} |`
+    )
+  })
+  const parts = packLines(expanded, rowBudget, location)
+  return parts.map(
+    (part, index) => `${excerptLabel('Table', index + 1, parts.length)}${prefix}${part}`
+  )
+}
+
+function blockParts(node: RootContent, raw: string, limit: number, location: string) {
+  if (raw.length <= limit) return [raw]
+  if (node.type === 'code') return codeParts(node, raw, limit, location)
+  const lines = raw.split('\n')
+  if (lines[0]?.includes('|') && /^[\s|:-]+$/.test(lines[1] ?? '') && lines[1]?.includes('-'))
+    return tableParts(raw, limit, location)
+  let nestedCode = false
+  visit(node, 'code', () => {
+    nestedCode = true
+  })
+  if (nestedCode)
+    throw new Error(`${location}: oversized nested code requires a separate search block`)
+  return splitText(raw, limit, location)
+}
+
+function sectionChunks(text: string, page: DocsPage, heading: DocsHeading): string[] {
+  if (!text) return []
+  if (text.length <= CHUNK_CHARACTERS) return [text]
+  const context = `# ${page.title}\n\n${'#'.repeat(heading.depth)} ${heading.title} (continued)\n\n`
+  const limit = CHUNK_CHARACTERS - context.length
+  const location = `${page.url}${heading.id ? `#${heading.id}` : ''}`
+  if (limit < 100) throw new Error(`${location}: heading context exceeds search chunk budget`)
+  const chunks: string[] = []
+  let current = ''
+  const flush = () => {
+    if (current) chunks.push(`${chunks.length ? context : ''}${current}`)
+    current = ''
+  }
+  for (const node of parser.parse(text).children) {
+    const raw = text.slice(node.position?.start.offset, node.position?.end.offset)
+    for (const part of blockParts(node, raw, limit, location)) {
+      if (current && current.length + 2 + part.length > limit) flush()
+      current += `${current ? '\n\n' : ''}${part}`
+    }
+  }
+  flush()
+  return chunks
+}
+
 /** Chunk processed Markdown by AST headings, preserving catalog IDs rather than generating slugs. */
 export function* pageSearchChunks(page: DocsPage): Generator<SearchChunk> {
   const headingsByTitle = new Map<string, DocsHeading[]>()
@@ -33,9 +161,9 @@ export function* pageSearchChunks(page: DocsPage): Generator<SearchChunk> {
   })
   for (const [index, section] of sections.entries()) {
     const text = page.markdown.slice(section.offset, sections[index + 1]?.offset).trim()
-    for (let offset = 0; offset < text.length; offset += CHUNK_CHARACTERS) {
+    for (const [chunkIndex, chunk] of sectionChunks(text, page, section.heading).entries()) {
       yield {
-        id: `${page.id}:${section.heading.id}:${offset / CHUNK_CHARACTERS}`,
+        id: `${page.id}:${section.heading.id}:${chunkIndex}`,
         pageId: page.id,
         sourceDocument: page.operation
           ? `${page.url.slice(1)}.mdx`
@@ -46,7 +174,7 @@ export function* pageSearchChunks(page: DocsPage): Generator<SearchChunk> {
         ).href,
         headerText: section.heading.title,
         headerLevel: section.heading.depth,
-        text: text.slice(offset, offset + CHUNK_CHARACTERS),
+        text: chunk,
         contentHash: page.contentHash,
         documentTitle: page.title,
       }

apps/docs/next.config.ts

@@ -6,6 +6,7 @@ const config: NextConfig = {
   transpilePackages: ['@sim/emcn', '@sim/workflow-renderer'],
   images: {
     deviceSizes: [384, 640, 828, 1080, 1440, 1920, 2560],
+    imageSizes: [32, 48, 64, 96, 128, 256],
     qualities: [75, 90],
     minimumCacheTTL: 60,
     localPatterns: [{ pathname: '/_next/static/media/**' }, { pathname: '/static/**' }],

apps/docs/scripts/benchmark-readiness.ts

@@ -941,7 +941,7 @@ async function interactions(browser: Browser, profile: Profile, repetition: numb
         .getByRole('button', { name: /^(Expand .*workflow preview|Inspect workflow: .*)$/i })
         .first()
       await trigger.waitFor({ state: 'visible', timeout: 15_000 })
-      await trigger.scrollIntoViewIfNeeded()
+      await trigger.click({ trial: true })
       const start = performance.now()
       await trigger.click()
       const dialog = run.page.getByRole('dialog').first()

apps/docs/scripts/test-negotiation-e2e.ts

@@ -21,10 +21,20 @@ interface CheckResult {
   durationMs: number
   contentType?: string
   vary?: string
+  statusCode?: number
   error?: string
 }
 
-const cases = [
+interface RepresentationCase {
+  name: string
+  accept: string
+  expected: 'text/html' | 'text/markdown'
+  method?: 'GET' | 'HEAD'
+  statusCode?: number
+  useHtmlValidator?: boolean
+}
+
+const cases: readonly RepresentationCase[] = [
   {
     name: 'HTML advertises its representation and router cache dimensions',
     accept: 'text/html',
@@ -60,6 +70,39 @@ const cases = [
     accept: 'text/html;q=0,text/*;q=0.5',
     expected: 'text/markdown',
   },
+  {
+    name: 'HTML HEAD retains representation headers without a body',
+    accept: 'text/html',
+    expected: 'text/html',
+    method: 'HEAD',
+  },
+  {
+    name: 'Markdown HEAD retains representation headers without a body',
+    accept: 'text/markdown',
+    expected: 'text/markdown',
+    method: 'HEAD',
+  },
+  {
+    name: 'Matching HTML validator returns a bodyless 304 with cache dimensions',
+    accept: 'text/html',
+    expected: 'text/html',
+    statusCode: 304,
+    useHtmlValidator: true,
+  },
+  {
+    name: 'Conditional HTML HEAD returns a bodyless 304',
+    accept: 'text/html',
+    expected: 'text/html',
+    method: 'HEAD',
+    statusCode: 304,
+    useHtmlValidator: true,
+  },
+  {
+    name: 'An HTML validator cannot suppress the Markdown representation',
+    accept: 'text/markdown',
+    expected: 'text/markdown',
+    useHtmlValidator: true,
+  },
 ] as const
 
 for (let attempt = 0; attempt < 20; attempt += 1) {
@@ -72,24 +115,37 @@ for (let attempt = 0; attempt < 20; attempt += 1) {
 }
 
 const checks: CheckResult[] = []
+let htmlValidator: string | undefined
 for (const entry of cases) {
   const started = performance.now()
   const result: CheckResult = { name: entry.name, status: 'failed', durationMs: 0 }
   try {
+    const headers = new Headers({ Accept: entry.accept })
+    if (entry.useHtmlValidator) {
+      if (!htmlValidator) throw new Error('HTML did not provide a cache validator')
+      headers.set('If-None-Match', htmlValidator)
+    }
     const response = await fetch(url, {
-      headers: { Accept: entry.accept },
+      method: entry.method ?? 'GET',
+      headers,
       signal: AbortSignal.timeout(10000),
     })
-    await response.body?.cancel()
+    if (entry.method === 'HEAD' || entry.statusCode === 304) {
+      if ((await response.arrayBuffer()).byteLength !== 0)
+        throw new Error('HEAD and 304 responses must not contain a body')
+    } else await response.body?.cancel()
+    result.statusCode = response.status
     result.contentType = response.headers.get('Content-Type') ?? ''
     result.vary = response.headers.get('Vary') ?? ''
     const fields = result.vary.split(',').map((field) => field.trim().toLowerCase())
-    if (!response.ok) throw new Error(`Unexpected HTTP status ${response.status}`)
-    if (!result.contentType.startsWith(entry.expected)) {
+    if (response.status !== (entry.statusCode ?? 200))
+      throw new Error(`Unexpected HTTP status ${response.status}`)
+    if (response.status !== 304 && !result.contentType.startsWith(entry.expected)) {
       throw new Error(`Expected ${entry.expected}; received ${result.contentType}`)
     }
     if (!fields.includes('accept')) throw new Error('The response lost Vary: Accept')
     if (entry.expected === 'text/html') {
+      if (response.status === 200) htmlValidator = response.headers.get('etag') ?? undefined
       for (const field of [
         'rsc',
         'next-router-state-tree',