diff --git a/website/.gitignore b/website/.gitignore index b2d2e08dac..56a18ec13e 100644 --- a/website/.gitignore +++ b/website/.gitignore @@ -1,5 +1,6 @@ # Dependencies /node_modules +/docsearch-export/ npm-debug.log* yarn-debug.log* diff --git a/website/.prettierignore b/website/.prettierignore index bd5535a603..468ed1aa64 100644 --- a/website/.prettierignore +++ b/website/.prettierignore @@ -1 +1,2 @@ pnpm-lock.yaml +docsearch-export/ diff --git a/website/.vitepress/config.ts b/website/.vitepress/config.ts index b11bb5c438..68530a9ba3 100644 --- a/website/.vitepress/config.ts +++ b/website/.vitepress/config.ts @@ -1,6 +1,7 @@ import { defineConfig, HeadConfig } from 'vitepress'; import githubLinksPlugin from './plugins/github-links'; import { renderSearchContent } from './plugins/local-search'; +import { searchKeywordsPlugin } from './plugins/search-keywords'; import { existsSync, readdirSync, readFileSync, writeFileSync } from 'fs'; import { resolve } from 'path'; import matter from 'gray-matter'; @@ -165,6 +166,12 @@ export default defineConfig({ { name: 'docsearch:doc_type', content: pageData.frontmatter.docType } ]); } + if ( + pageData.frontmatter.search === false || + pageData.frontmatter.noindex === true + ) { + head.push(['meta', { name: 'docsearch:exclude', content: 'true' }]); + } // Dynamic Open Graph and Twitter meta tags const isHome = new URL(canonicalUrl).pathname === '/'; @@ -334,6 +341,7 @@ export default defineConfig({ }); md.use(tabsMarkdownPlugin); md.use(groupIconMdPlugin); + md.use(searchKeywordsPlugin); } }, vite: { @@ -383,13 +391,13 @@ export default defineConfig({ code: 'CESI65QJ', placement: 'taskfiledev' }, - search: isProduction + search: isPublicDeploy ? { provider: 'algolia', options: { appId: '7IZIJ13AI7', apiKey: '34b64ae4fc8d9da43d9a13d9710aaddc', - indexName: 'taskfile' + indexName: isLatest ? 'taskfile' : 'taskfile-next' } } : { @@ -483,7 +491,7 @@ export default defineConfig({ } }, sitemap: { - hostname: 'https://taskfile.dev' + hostname: isLatest ? 'https://taskfile.dev' : 'https://next.taskfile.dev' }, buildEnd({ outDir }) { const robots = isProduction @@ -494,7 +502,21 @@ export default defineConfig({ 'Sitemap: https://taskfile.dev/sitemap.xml', '' ] - : ['User-agent: *', 'Disallow: /', '']; + : isPublicDeploy + ? [ + 'User-agent: Algolia Crawler', + 'Allow: /docs/', + 'Allow: /sitemap.xml', + 'Allow: /$', + 'Disallow: /', + '', + 'User-agent: *', + 'Disallow: /', + '', + 'Sitemap: https://next.taskfile.dev/sitemap.xml', + '' + ] + : ['User-agent: *', 'Disallow: /', '']; writeFileSync(resolve(outDir, 'robots.txt'), robots.join('\n')); } }); diff --git a/website/.vitepress/plugins/local-search.ts b/website/.vitepress/plugins/local-search.ts index 2addaa89fd..f5102e8b45 100644 --- a/website/.vitepress/plugins/local-search.ts +++ b/website/.vitepress/plugins/local-search.ts @@ -8,6 +8,13 @@ export function renderSearchContent( let html = md.render(src, env); if (env.frontmatter?.search === false) return ''; + // Only this index renderer adds the alternate terms to section content. + // The actual page keeps them in an attribute, invisible to readers. + html = html.replace( + /(]*\bdata-search-keywords="([^"]*)"[^>]*>[\s\S]*?<\/h[1-6]>)/g, + '$1

$2

' + ); + // Otherwise stripping the tags joins the language label to the first // command ("shellbrew"), preventing searches for "brew install". html = html.replace(/[^<]*<\/span>/g, ''); diff --git a/website/.vitepress/plugins/search-keywords.ts b/website/.vitepress/plugins/search-keywords.ts new file mode 100644 index 0000000000..f619caae4b --- /dev/null +++ b/website/.vitepress/plugins/search-keywords.ts @@ -0,0 +1,36 @@ +import type MarkdownIt from 'markdown-it'; + +// Keep alternate search terms attached to the heading they describe. HTML +// attributes don't change the visible heading, its permalink or the outline. +export function searchKeywordsPlugin(md: MarkdownIt): void { + md.core.ruler.push('search-keywords', (state) => { + const keywords = state.env.frontmatter?.searchKeywords; + if (keywords === undefined) return; + const fail = (message: string): never => { + throw new Error( + `searchKeywords (${state.env.relativePath ?? 'page'}): ${message}` + ); + }; + if (!keywords || typeof keywords !== 'object' || Array.isArray(keywords)) { + fail('expected a mapping of heading IDs to lists of search terms'); + } + + const headings = new Map( + state.tokens + .filter((token) => token.type === 'heading_open') + .map((token) => [token.attrGet('id'), token]) + ); + for (const [anchor, terms] of Object.entries(keywords)) { + const heading = headings.get(anchor); + if (!heading) return fail(`heading #${anchor} does not exist`); + if ( + !Array.isArray(terms) || + !terms.length || + terms.some((term) => typeof term !== 'string' || !term.trim()) + ) { + return fail(`#${anchor} needs a non-empty list of strings`); + } + heading.attrSet('data-search-keywords', terms.join(' ')); + } + }); +} diff --git a/website/docsearch.config.js b/website/docsearch.config.js index 294411c68b..c5a29ba033 100644 --- a/website/docsearch.config.js +++ b/website/docsearch.config.js @@ -1,16 +1,204 @@ -// Algolia Crawler configuration for taskfile.dev. +// Run `pnpm docsearch:export` from website/ to generate the dashboard configuration +// and per-index settings in docsearch-export/. Preserve the crawler's write key +// when pasting the generated configuration. Apply changed settings to existing +// indices separately; initialIndexSettings only initializes new indices. // -// This file is the source of truth. The crawler itself runs on Algolia's side, -// configured through the dashboard at https://crawler.algolia.com, and until now -// nothing described it here: `git log --all -- '*algolia*' '*docsearch*'` -// returned nothing, so the only copy lived in a web form nobody could review. -// When you change the configuration in the dashboard, change it here too. -// -// The API key below is the crawler's *write* key and is deliberately not in -// this repository. Keep the existing key when pasting this file into the -// dashboard; never replace this placeholder in Git. -// -// Every selector here was checked against the generated HTML, not assumed. +// Both actions share an extractor, but write to separate indices. Keep the +// extractor self-contained: the hosted crawler can serialize it independently. + +const channels = [ + { origin: 'https://taskfile.dev', indexName: 'taskfile' }, + { origin: 'https://next.taskfile.dev', indexName: 'taskfile-next' } +]; + +const recordExtractor = ({ $, helpers, url }) => { + // ignoreNoIndex is needed for the next channel. Still honor page-level + // opt-outs and the released site's noindex directives. + if ($('meta[name="docsearch:exclude"]').attr('content') === 'true') return []; + if ( + url.hostname === 'taskfile.dev' && + /\b(noindex|none)\b/i.test($('meta[name="robots"]').attr('content') || '') + ) + return []; + // The banner the llms plugin injects sits inside .vp-doc, ahead of the + // h1. It is display:none for readers and must not become content. + $('[data-nosnippet]').remove(); + + // DocSearch expects content selectors to target paragraphs or list + // items. Copy code blocks into crawler-only paragraphs so experienced + // users can search for exact Taskfile keys and command syntax without + // changing the page rendered to readers. + $('.vp-doc pre code').each((_, element) => { + const code = $(element).text().trim(); + if (!code) return; + const paragraph = $('

').addClass('docsearch-code').text(code); + $(element).closest('pre').replaceWith(paragraph); + }); + + // Select each text subtree once. A list item may contain paragraphs and + // nested lists; selecting both parent and child repeats their content. + $('.vp-doc p, .vp-doc li, .vp-doc td, .vp-doc th').each((_, element) => { + if (!$(element).parents('.docsearch-content').length) { + $(element).addClass('docsearch-content'); + } + }); + + // Frontmatter metadata is available after the refactor. Infer the same + // values from the URL while the old monolithic guide is still live, so + // this configuration can be installed before the website PR merges. + // Remove this URL inference once the refactored documentation is live + // and every indexed page exposes the DocSearch metadata. + const pathname = url.pathname.replace(/\/+$/, '') || '/'; + const inferredSection = (() => { + if (pathname === '/docs') return 'Overview'; + if ( + /^\/docs\/(installation|getting-started|integrations)$/.test(pathname) + ) { + return 'Getting Started'; + } + if (/^\/docs\/reference\//.test(pathname)) return 'Reference'; + if (/^\/docs\/(contributing|releasing|styleguide)$/.test(pathname)) { + return 'Contributing'; + } + if ( + /^\/docs\/(experiments|deprecations|security)(\/|$)/.test(pathname) || + /^\/docs\/(changelog|faq|taskfile-versions|community)$/.test(pathname) + ) { + return 'Project'; + } + return 'Guide'; + })(); + const section = + $('meta[name="docsearch:section"]').attr('content') || inferredSection; + const docType = + $('meta[name="docsearch:doc_type"]').attr('content') || + ({ + Overview: 'overview', + Reference: 'reference', + Contributing: 'contributing', + Project: 'project', + Guide: 'guide', + 'Getting Started': 'guide' + }[section] ?? + 'guide'); + + const keywordsByAnchor = new Map(); + $('.vp-doc [data-search-keywords]').each((_, element) => { + const heading = $(element); + keywordsByAnchor.set( + heading.attr('id'), + heading.attr('data-search-keywords') + ); + }); + + const records = helpers.docsearch({ + recordProps: { + // Not a heading on the page: the section the page belongs to, + // stated in its own frontmatter and emitted by transformHead. The + // usual DocSearch recipe reads the active sidebar link out of the + // DOM instead, which ties the index to the theme's markup and + // breaks silently when that markup changes. + lvl0: { + // Algolia documents an empty selector as the way to provide a + // raw, dynamically computed lvl0 through defaultValue. + selectors: '', + defaultValue: section + }, + // Everything below is scoped to .vp-doc. VitePress renders the + // sidebar's section labels as

inside + //