Make API references searchable and easier to scan

packages/api-model/src/builtins.ts

@@ -77,8 +77,6 @@ export const BUILTINS: Record<string, Record<string, string>> = {
   },
   scala: {
     ...qualifiedScala,
-    def: 'https://docs.scala-lang.org/scala3/book/methods-most.html',
-    extension: 'https://docs.scala-lang.org/scala3/reference/contextual/extension-methods.html',
     String: scala('scala/Predef$'), Boolean: scala('scala/Boolean'),
     Int: scala('scala/Int'), Long: scala('scala/Long'), Unit: scala('scala/Unit'),
     Option: scala('scala/Option'), Either: scala('scala/util/Either'),

packages/api-model/src/link.ts

@@ -1,6 +1,6 @@
 import { tokenizeDoc, type DocSpan } from './doc/roles.ts'
 import type { InventoryEntry } from './inventory.ts'
-import type { ApiSymbol, SymbolKind } from './model.ts'
+import type { ApiSymbol, Signature, SymbolKind } from './model.ts'
 import { builtinHref } from './builtins.ts'
 
 /**
@@ -65,6 +65,14 @@ const PY_INTERSPHINX: Record<string, string> = {
 
 const PY_DOCS = 'https://docs.python.org/3/'
 
+/** Syntax words in the native declarations rendered as complete signatures. */
+const SIGNATURE_KEYWORDS: Record<string, Set<string>> = Object.fromEntries(Object.entries({
+  kotlin: 'public private protected internal class interface object fun val var suspend override abstract open final data sealed enum inline reified crossinline noinline vararg in out where companion constructor operator infix tailrec external expect actual by',
+  scala: 'def extension val var class trait object type given using implicit inline transparent opaque override abstract final sealed case lazy private protected open infix export derives end',
+  fsharp: 'module namespace type member static abstract override interface inherit let rec mutable inline internal private public of with get set new val and when',
+  ts: 'readonly keyof typeof infer extends unique',
+}).map(([port, words]) => [port, new Set(words.split(' '))]))
+
 /**
  * Builtin exceptions, which docstrings reference constantly and which all live
  * on one page.
@@ -379,6 +387,11 @@ export class SymbolIndex {
       if (byName) return { href: this.hrefFor(byName), external: false, symbol: byName }
     }
 
+    // Scala and Kotlin default imports take precedence over a JDK inventory's
+    // bare aliases. An explicit java.util.Vector import still resolves there.
+    const builtin = this.lang ? builtinHref(this.lang, clean) : undefined
+    if (builtin && ['kotlin', 'scala'].includes(this.lang ?? '')) return { href: builtin, external: true }
+
     // Intersphinx order: exact name, then the shortest suffix. An inventory
     // is authoritative for its own project, so it is consulted before the
     // built-in table rather than after.
@@ -389,7 +402,6 @@ export class SymbolIndex {
       if (hit) return { href: inv.baseUrl + hit.uri, external: true, project: inv.project }
     }
 
-    const builtin = this.lang ? builtinHref(this.lang, clean) : undefined
     if (builtin) return { href: builtin, external: true }
 
     // Everything below is CPython's, and applies to CPython only. `time`,
@@ -446,8 +458,8 @@ export class SymbolIndex {
    * identifier boundaries and passing everything else through verbatim is what
    * makes those all work without a grammar for type syntax.
    */
-  linkType(annotation: string, context?: ApiSymbol): { text: string; link?: LinkTarget; declaration?: true }[] {
-    const out: { text: string; link?: LinkTarget; declaration?: true }[] = []
+  linkType(annotation: string, context?: ApiSymbol, signature?: Signature): { text: string; link?: LinkTarget; declaration?: true; keyword?: true }[] {
+    const out: { text: string; link?: LinkTarget; declaration?: true; keyword?: true }[] = []
     // Identifiers, including dotted ones; everything else is punctuation,
     // whitespace or a string literal and passes through untouched.
     // `::` is part of a name, not punctuation between two. Splitting there
@@ -468,6 +480,28 @@ export class SymbolIndex {
         out.push({ text: ident })
         continue
       }
+      // A declaration's bindings are not references to other APIs. Keep this
+      // separate from prose resolution, where an explicit parameter reference
+      // can still point to that parameter's documentation.
+      if (SIGNATURE_KEYWORDS[this.lang ?? '']?.has(ident)) {
+        out.push({ text: ident, keyword: true })
+        continue
+      }
+      if (signature && context && (ident === context.name || ident.endsWith(`.${context.name}`)) &&
+          /\b(?:def|fun|class|interface|trait|object|type|val|var|member|let)\s+$/.test(annotation.slice(0, m.index))) {
+        const receiver = ident.slice(0, -context.name.length).replace(/\.$/, '')
+        if (receiver) out.push(...this.linkType(receiver, context), { text: '.' })
+        out.push({ text: context.name, declaration: true })
+        continue
+      }
+      if (signature && (
+        ((signature.params.some((param) => param.name === ident) || signature.receiver?.name === ident) &&
+          /^\s*\??\s*:(?!:)/.test(annotation.slice(m.index! + ident.length))) ||
+        signature.typeParams?.includes(ident)
+      )) {
+        out.push({ text: ident, declaration: true })
+        continue
+      }
       // Object fields, callback parameters and tuple labels declare names;
       // the types after their colons still use ordinary reference resolution.
       if (this.lang === 'ts' && /^\s*\??\s*:(?!:)/.test(annotation.slice(m.index! + ident.length)) &&

packages/api-model/src/nav-config.ts

@@ -186,7 +186,7 @@ const CHAIN: { id: string; label: string; match: Match }[] = [
     match: {
       kind: 'anyOf',
       of: [
-        nameRe('Internal|Generated'),
+        nameRe('Internal|Generated|^__fuzz_'),
         // A public API can live in a private implementation directory. Lua
         // exposes Server, Session, Window, Pane and Client from `_internal/`;
         // their public contract outranks that layout detail while retained

packages/api-model/test/builtins.test.ts

@@ -40,4 +40,15 @@ describe('standard types in product signatures', () => {
     expect(signature.find((span) => span.text === 'toString')?.link).toBeUndefined()
     expect(signature.find((span) => span.text === 'String')?.link?.href).toMatch(/^https:\/\//)
   })
+
+  it('uses Scala collections before bare aliases from the JDK inventory', () => {
+    const index = new SymbolIndex([], () => '#', 'scala')
+    index.addInventory('https://docs.oracle.com/', ['Vector', 'java.util.Vector'].map((name) => ({
+      name, type: 'class', priority: 1, uri: 'java/util/Vector.html', dispname: '-',
+    })), ['scala'], 'Java SE')
+    expect(index.resolve('Vector')?.href).toBe('https://www.scala-lang.org/api/3.x/scala/collection/immutable/Vector.html')
+    expect(index.resolve('java.util.Vector')?.href).toBe('https://docs.oracle.com/java/util/Vector.html')
+    expect(index.resolve('def')).toBeUndefined()
+    expect(index.resolve('extension')).toBeUndefined()
+  })
 })

packages/api-model/test/jvm.test.ts

@@ -36,6 +36,28 @@ extension [F[_]](self: Box[F, String]) {
     expect(index.resolve('B', 'class', map)?.href).toBe('#example.Box.map')
     expect(index.resolve('self', 'any', map)?.href).toBe('#example.Box')
     expect(index.resolve('UnresolvedType', 'class', map)).toBeUndefined()
+    const signature = map.signatures[0]
+    const spans = index.linkType(signature.raw!, map, signature)
+    expect(spans.filter((span) => span.link).map((span) => span.text)).toEqual(['Box', 'String'])
+    expect(spans.map((span) => span.text).join('')).toBe(signature.raw)
+  })
+  it('links the types in an overload while leaving its bindings plain', async () => {
+    const root = fixture('Server.scala', `package example
+class Server
+class Window
+class WindowId
+extension (self: Server) {
+  def windows(id: WindowId): Vector[Window] = TODO
+}
+`)
+    const { symbols } = await extractJvm('scala', [root])
+    const method = symbols.find((s) => s.name === 'windows')!
+    const index = new SymbolIndex(symbols, (symbol) => `#${symbol.id}`, 'scala')
+    const signature = method.signatures[0]
+    const spans = index.linkType(signature.raw!, method, signature)
+    expect(spans.filter((span) => span.link).map((span) => span.text)).toEqual(['Server', 'WindowId', 'Vector', 'Window'])
+    expect(spans.map((span) => span.text).join('')).toBe(signature.raw)
+    expect(index.resolve('id', 'any', method)?.href).toContain('.parameter.id')
   })
   it.each(['kotlin', 'scala'] as const)('keeps %s links outside generic code', async (port) => {
     const doc = port === 'kotlin' ? '[the server][Handle]' : '[[Handle the server]]'
@@ -89,6 +111,11 @@ public val KotlinPane.Companion.id: TextField<JavaPane>
     expect(symbols.some((s) => /\.java$|\.implementation$/.test(s.id))).toBe(false)
     expect(symbols.find((s) => s.id === 'example.Pane')!.signatures[0]).toMatchObject({ raw: 'public class Pane', params: [] })
     const capture = symbols.find((s) => s.id === 'example.Pane.capture')!
+    const index = new SymbolIndex(symbols, (symbol) => `#${symbol.id}`, 'kotlin')
+    const signature = capture.signatures[0]
+    const spans = index.linkType(signature.raw!, capture, signature)
+    expect(spans.filter((span) => span.link).map((span) => span.text)).toEqual(['Pane', 'Boolean', 'List', 'String'])
+    expect(spans.map((span) => span.text).join('')).toBe(signature.raw)
     expect(capture.parent).toBe('example.Pane')
     expect(capture.modifiers).toEqual(['async', 'overload'])
     expect(capture.signatures).toHaveLength(2)

scripts/check-api-fidelity.mjs

@@ -34,6 +34,7 @@ if (!root) {
 const { API_MODEL_PORTS: PORT_DEFS } = await import(`file://${join(repoRoot, 'site/src/lib/ports.ts')}`)
 const PORTS = PORT_DEFS.map((p) => p.slug)
 const HOOKS = [
+  'data-symbol-id',
   'data-domain',
   'data-objtype',
   'data-badge-count',
@@ -83,6 +84,8 @@ function entriesIn(html) {
 
 const failures = []
 const summary = []
+const decode = (value) => value?.replace(/&(amp|lt|gt|quot|#39);/g, (_, entity) =>
+  ({ amp: '&', lt: '<', gt: '>', quot: '"', '#39': "'" })[entity])
 
 for (const port of PORTS) {
   const model = JSON.parse(readFileSync(join(repoRoot, `site/src/data/api/${port}.json`), 'utf8'))
@@ -110,9 +113,8 @@ for (const port of PORTS) {
 
     for (const attrs of entriesIn(html)) {
       entries++
-      const id = attrs.id?.replace(/&(amp|lt|gt|quot|#39);/g, (_, entity) =>
-        ({ amp: '&', lt: '<', gt: '>', quot: '"', '#39': "'" })[entity])
-      const symbol = symbols.get(id)
+      const id = decode(attrs.id)
+      const symbol = symbols.get(decode(attrs['data-symbol-id']))
       if (!symbol?.inheritedFrom || symbol.source.file) eligible++
       if (symbol && sourceUrl(model, symbol, '/reference/') && attrs['data-has-source'] !== 'true') {
         failures.push(`${port}: ${rel}#${id} omits its known source link`)

scripts/normalize-native-shell.mjs

@@ -54,6 +54,20 @@ export async function normalizeNativeShell(directory, prefix, { sphinxPort, vers
       } else if (/\.(html|css)$/.test(entry.name)) {
         const before = readFileSync(path, 'utf8')
         let after = normalize(before)
+        // Native search delegates to the owned search page for this port/version.
+        if (port && /^search(?:\.html|\/index\.html)$/.test(relative(directory, path).split(sep).join('/'))) {
+          const target = `${root}/${port.slug}/${version}/search/`
+          const href = escapeAttribute(target)
+          const scriptTarget = JSON.stringify(target).replaceAll('<', '\\u003c')
+          after = `<!doctype html>
+<html><head><meta charset="utf-8"><title>Search</title>
+<meta name="robots" content="noindex, follow">
+<link rel="canonical" href="${href}">
+<script>window.location.replace(${scriptTarget} + window.location.search + window.location.hash)</script>
+<meta http-equiv="refresh" content="0; url=${href}">
+</head><body><p><a href="${href}">Search the ${port.name} documentation</a>.</p></body></html>
+`
+        }
         if (port && entry.name.endsWith('.html') && !/<meta\b[^>]*http-equiv=["']refresh["']/i.test(after)) {
           if (!/<\/head>/i.test(after)) throw new Error(`Native page has no closing head: ${path}`)
           // Old source refs may predate the shell; replace existing integration once.