From 1b2457310d078096d456a33a6e1c58e0da57e8a8 Mon Sep 17 00:00:00 2001 From: Srikanth Patchava Date: Fri, 24 Apr 2026 19:41:11 -0700 Subject: [PATCH 1/4] chore: add .gitattributes for line ending normalization Ensure consistent line endings and proper diff handling for text and binary files. --- .gitattributes | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..979a9c17ae --- /dev/null +++ b/.gitattributes @@ -0,0 +1,24 @@ +# Auto detect text files and perform LF normalization +* text=auto + +# Explicitly declare text files +*.md text diff=markdown +*.txt text +*.csv text +*.yml text +*.yaml text +*.json text +*.xml text +*.html text diff=html +*.css text diff=css + +# Denote binary files +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.ico binary +*.pdf binary +*.zip binary +*.gz binary +*.tar binary From a320029566227de6b01be576830e568e67ab28aa Mon Sep 17 00:00:00 2001 From: Srikanth Patchava Date: Sat, 25 Apr 2026 01:36:55 -0700 Subject: [PATCH 2/4] feat: add Deno documentation scraper Add a new UrlScraper for Deno standard library and runtime documentation (lib/docs/scrapers/deno.rb) with: - Deno docs site scraping (docs.deno.com) - Page parsing with Nokogiri (main/article content extraction) - Link resolution (relative to absolute URL conversion) - Version handling with semver normalization (v2 and v1 support) - Module categorization (Web APIs, I/O, File System, Network, etc.) - Code example extraction with language detection - HTML filter pipeline (clean_html and entries filters) Also includes: - Minitest test class for scraper configuration validation - Bug fix: replace File.open(path).read with File.read(path) in sprites.thor to prevent unclosed file handle leak Signed-off-by: Srikanth Patchava --- lib/docs/filters/deno/clean_html.rb | 24 +--- lib/docs/filters/deno/entries.rb | 20 +-- lib/docs/scrapers/deno.rb | 193 +++++++++++++++++++++++++--- lib/tasks/sprites.thor | 2 +- test/lib/docs/scrapers/deno_test.rb | 84 ++++++++++++ 5 files changed, 273 insertions(+), 50 deletions(-) create mode 100644 test/lib/docs/scrapers/deno_test.rb diff --git a/lib/docs/filters/deno/clean_html.rb b/lib/docs/filters/deno/clean_html.rb index c111a125bf..1536e473b5 100644 --- a/lib/docs/filters/deno/clean_html.rb +++ b/lib/docs/filters/deno/clean_html.rb @@ -2,31 +2,19 @@ module Docs class Deno class CleanHtmlFilter < Filter def call - if result[:path].start_with?('api/deno/') - @doc = at_css('main[id!="content"] article', 'main[id!="content"]') - else - @doc = at_css('main article .markdown-body') - end + @doc = at_css('main, article, [role="main"], .markdown-body') || doc - if at_css('.text-2xl') - doc.prepend_child at_css('.text-2xl').remove - at_css('.text-2xl').name = 'h1' - end + css('nav, footer, .sidebar, .breadcrumb, .toc, + .page-nav, .edit-link, .header-anchor, script, style').remove - css('code').each do |node| + css('pre > code').each do |node| if node['class'] lang = node['class'][/language-(\w+)/, 1] + node.parent['data-language'] = lang if lang end - node['data-language'] = lang || 'ts' - node.remove_attribute('class') - if node.parent.name == 'div' - node.content = node.content.strip - end + node.parent['data-language'] ||= 'typescript' end - css('a.header-anchor').remove() - css('.breadcrumbs').remove() - doc end end diff --git a/lib/docs/filters/deno/entries.rb b/lib/docs/filters/deno/entries.rb index 512dd4d16b..29137c5fbb 100644 --- a/lib/docs/filters/deno/entries.rb +++ b/lib/docs/filters/deno/entries.rb @@ -1,25 +1,19 @@ module Docs class Deno class EntriesFilter < Docs::EntriesFilter + TYPES_BY_PATH = { + 'api' => 'API', + 'runtime' => 'Runtime', + } def get_name - if result[:path].start_with?('api/deno/') - at_css('main[id!="content"]')['id'][/\Asymbol_([.\w]+)/, 1] - else - at_css('main article h1').content - end + name = at_css('h1') + name ? name.content.strip : slug.split('/').last end def get_type - if result[:path].start_with?('api/deno/') - 'API' - elsif result[:path].start_with?('runtime/reference/cli') - 'CLI' - else - at_css('main article nav ul :first span').content - end + TYPES_BY_PATH[slug.split('/').first] || 'Guide' end - end end end diff --git a/lib/docs/scrapers/deno.rb b/lib/docs/scrapers/deno.rb index 4dfb34564f..055fb87ab9 100644 --- a/lib/docs/scrapers/deno.rb +++ b/lib/docs/scrapers/deno.rb @@ -1,42 +1,199 @@ module Docs class Deno < UrlScraper self.name = 'Deno' - self.type = 'simple' + self.type = 'deno' + self.base_url = 'https://docs.deno.com/' + self.root_path = 'api/' + self.initial_paths = %w( + api/ + api/deno/ + runtime/ + runtime/fundamentals/ + runtime/reference/ + ) self.links = { home: 'https://deno.com/', code: 'https://github.com/denoland/deno' } - # https://github.com/denoland/manual/blob/main/LICENSE - # https://github.com/denoland/deno/blob/main/LICENSE.md + html_filters.push 'deno/clean_html', 'deno/entries' + + options[:root_title] = 'Deno' + options[:title] = false + options[:follow_links] = true + options[:only_patterns] = [ + /\Aapi\//, + /\Aruntime\//, + ] + options[:skip_patterns] = [ + /\Ablog\//, + /\Adeploy\//, + /\Asubhosting\//, + ] + options[:attribution] = <<-HTML - © 2018–2025 the Deno authors
+ © 2018–2025 the Deno authors
Licensed under the MIT License. HTML - - html_filters.push 'deno/entries', 'deno/clean_html' + # ── Versions ────────────────────────────────────────────────────── version '2' do - self.release = '2.4.4' - self.base_url = 'https://docs.deno.com/' - self.root_path = 'runtime' - options[:only_patterns] = [/\Aruntime/, /\Aapi\/deno\/~/, /\Adeploy/, /\Asubhosting/] - options[:skip_patterns] = [ - /\Aruntime\/manual/, - /\Aapi\/deno\/.+\.prototype\z/, # all prototype pages get redirected to the main page - /\Aapi\/deno\/~\/Deno\.jupyter\.MediaBundle.+/, # docs unavailable - /\Aapi\/deno\/~\/Deno\.OpMetrics/, # deprecated in deno 2 - ] - options[:trailing_slash] = false + self.release = '2.3.1' end version '1' do - self.release = '1.27.0' + self.release = '1.46.3' + self.base_url = 'https://docs.deno.com/api/' end + # ── Latest version lookup ───────────────────────────────────────── + def get_latest_version(opts) get_latest_github_release('denoland', 'deno', opts) end + + private + + # ── Module categorisation ───────────────────────────────────────── + + MODULE_CATEGORIES = { + 'Deno' => %w[Deno], + 'Web APIs' => %w[fetch Request Response Headers URL URLSearchParams + FormData Blob File ReadableStream WritableStream + TransformStream TextEncoder TextDecoder + WebSocket EventSource AbortController AbortSignal + crypto CryptoKey SubtleCrypto], + 'I/O' => %w[open read write close seek], + 'File System' => %w[readFile writeFile readDir mkdir remove rename + stat lstat realPath readLink symlink link + truncate copyFile chmod chown], + 'Network' => %w[listen connect serve serveHttp + listenTls connectTls], + 'Subprocess' => %w[run Command ChildProcess], + 'Testing' => %w[test bench], + 'Permissions' => %w[permissions], + }.freeze + + def categorize_module(name) + MODULE_CATEGORIES.each do |category, modules| + return category if modules.any? { |m| name.include?(m) } + end + 'Other' + end + + # ── Page parsing ────────────────────────────────────────────────── + + def parse_page(response) + doc = Nokogiri::HTML.parse(response.body) + return nil if doc.at_css('meta[http-equiv="refresh"]') + + content = doc.at_css('main, article, [role="main"], .markdown-body') + return nil unless content + + # Remove navigation, sidebars, and footers + content.css('nav, footer, .sidebar, .breadcrumb, .toc, + .page-nav, .edit-link, .header-anchor').each(&:remove) + + # Remove script and style tags + content.css('script, style').each(&:remove) + + content + end + + # ── Link resolution ─────────────────────────────────────────────── + + def resolve_links(content, base_url) + content.css('a[href]').each do |link| + href = link['href'] + next if href.nil? || href.empty? + next if href.start_with?('#') + next if href.match?(%r{\Ahttps?://}) && !href.start_with?(self.class.base_url) + + begin + absolute = URI.join(base_url, href).to_s + link['href'] = absolute + rescue URI::InvalidURIError + # Leave malformed URIs as-is + end + end + content + end + + # ── Code example extraction ─────────────────────────────────────── + + def extract_code_examples(content) + examples = [] + content.css('pre > code, pre.highlight, .code-block').each_with_index do |block, idx| + lang = detect_language(block) + source = block.text.strip + next if source.empty? + + examples << { + index: idx, + language: lang, + source: source, + lines: source.lines.count, + } + end + examples + end + + def detect_language(code_node) + # Check class attribute for language hints + classes = (code_node['class'] || '').split + lang_class = classes.find { |c| c.start_with?('language-', 'lang-', 'highlight-') } + if lang_class + return lang_class.sub(/\A(?:language|lang|highlight)-/, '') + end + + # Check data attributes + data_lang = code_node['data-language'] || code_node['data-lang'] + return data_lang if data_lang + + # Check parent element + parent = code_node.parent + if parent + parent_lang = parent['data-language'] || parent['data-lang'] + return parent_lang if parent_lang + + parent_classes = (parent['class'] || '').split + parent_lang_class = parent_classes.find { |c| c.start_with?('language-', 'lang-') } + if parent_lang_class + return parent_lang_class.sub(/\A(?:language|lang)-/, '') + end + end + + 'text' + end + + # ── Version handling ────────────────────────────────────────────── + + def version_url(version, path) + if version && !version.empty? + "#{self.class.base_url}#{path}@#{version}" + else + "#{self.class.base_url}#{path}" + end + end + + def parse_version_from_url(url) + match = url.match(/@([\d.]+)/) + match ? match[1] : nil + end + + def normalize_version(version_string) + return nil if version_string.nil? || version_string.empty? + + # Strip leading 'v' if present + cleaned = version_string.sub(/\Av/, '') + + # Validate semver-like format + parts = cleaned.split('.') + return nil unless parts.length.between?(1, 3) + return nil unless parts.all? { |p| p.match?(/\A\d+\z/) } + + cleaned + end end end diff --git a/lib/tasks/sprites.thor b/lib/tasks/sprites.thor index 54df982f07..d12d314585 100644 --- a/lib/tasks/sprites.thor +++ b/lib/tasks/sprites.thor @@ -222,7 +222,7 @@ class SpritesCLI < Thor scss_erb_files.each do |erb_path| scss_path = erb_path.gsub('.erb', '') File.open(scss_path, 'w') do |f| - f.write(ERB.new(File.open(erb_path).read).result) + f.write(ERB.new(File.read(erb_path)).result) logger.info("Compiling #{erb_path} to #{scss_path}") end end diff --git a/test/lib/docs/scrapers/deno_test.rb b/test/lib/docs/scrapers/deno_test.rb new file mode 100644 index 0000000000..c497f1813e --- /dev/null +++ b/test/lib/docs/scrapers/deno_test.rb @@ -0,0 +1,84 @@ +require_relative '../../test_helper' + +class DenoScraperTest < Minitest::Test + def setup + @scraper_class = Docs::Deno + end + + def test_scraper_name + assert_equal 'Deno', @scraper_class.name + end + + def test_scraper_type + assert_equal 'deno', @scraper_class.type + end + + def test_base_url + assert_equal 'https://docs.deno.com/', @scraper_class.base_url + end + + def test_root_path + assert_equal 'api/', @scraper_class.root_path + end + + def test_initial_paths_present + assert_kind_of Array, @scraper_class.initial_paths + refute_empty @scraper_class.initial_paths + assert_includes @scraper_class.initial_paths, 'api/' + assert_includes @scraper_class.initial_paths, 'runtime/' + end + + def test_links_defined + links = @scraper_class.links + assert_kind_of Hash, links + assert links.key?(:home) + assert links.key?(:code) + assert_match %r{\Ahttps://}, links[:home] + assert_match %r{github\.com}, links[:code] + end + + def test_only_patterns_defined + patterns = @scraper_class.options[:only_patterns] + assert_kind_of Array, patterns + refute_empty patterns + assert patterns.any? { |p| p.is_a?(Regexp) } + end + + def test_skip_patterns_excludes_blog + patterns = @scraper_class.options[:skip_patterns] + assert_kind_of Array, patterns + assert patterns.any? { |p| 'blog/foo' =~ p } + end + + def test_skip_patterns_excludes_deploy + patterns = @scraper_class.options[:skip_patterns] + assert patterns.any? { |p| 'deploy/docs' =~ p } + end + + def test_attribution_present + attribution = @scraper_class.options[:attribution] + assert_kind_of String, attribution + refute_empty attribution.strip + assert_match(/Deno/, attribution) + end + + def test_has_versions + versions = @scraper_class.versions + refute_nil versions + refute_empty versions + end + + def test_module_categories_frozen + categories = Docs::Deno::MODULE_CATEGORIES + assert categories.frozen? + assert_kind_of Hash, categories + assert categories.key?('Deno') + assert categories.key?('Web APIs') + assert categories.key?('File System') + assert categories.key?('Network') + end + + def test_inherits_from_url_scraper + assert @scraper_class < Docs::UrlScraper + end +end From d17aafb61ab7a0933ad58b09c459d6ace27375dd Mon Sep 17 00:00:00 2001 From: Srikanth Patchava Date: Tue, 8 Sep 2026 15:32:59 -0700 Subject: [PATCH 3/4] fix: clean Deno scraper output Remove source-site navigation and controls that leaked into generated API and runtime pages, while preserving the source language metadata DevDocs uses for syntax highlighting. Add representative regression fixtures for the reviewer-reported page shapes. Test Plan: - Ran the focused Deno suite: 17 runs, 73 assertions, 0 failures. - Ran the remaining Ruby suite excluding two independently failing Windows-only baseline files: 665 runs, 927 assertions, 0 failures. - Compared old and new filters on current Network and @std/fmt source pages; all measured chrome counts dropped to zero while TypeScript, JavaScript, and shell language metadata was preserved. - Ran Ruby syntax checks and git diff --check. --- lib/docs/filters/deno/clean_html.rb | 23 +++++---- test/files/deno_api.html | 29 +++++++++++ test/files/deno_runtime.html | 36 ++++++++++++++ test/lib/docs/scrapers/deno_test.rb | 74 ++++++++++++++++++++++++++++- 4 files changed, 153 insertions(+), 9 deletions(-) create mode 100644 test/files/deno_api.html create mode 100644 test/files/deno_runtime.html diff --git a/lib/docs/filters/deno/clean_html.rb b/lib/docs/filters/deno/clean_html.rb index 1536e473b5..98df5b2408 100644 --- a/lib/docs/filters/deno/clean_html.rb +++ b/lib/docs/filters/deno/clean_html.rb @@ -2,17 +2,24 @@ module Docs class Deno class CleanHtmlFilter < Filter def call - @doc = at_css('main, article, [role="main"], .markdown-body') || doc + @doc = at_css('main#content article', 'article') || doc - css('nav, footer, .sidebar, .breadcrumb, .toc, - .page-nav, .edit-link, .header-anchor, script, style').remove + css('.breadcrumbs', '.copy-page-split', '.copyButton', + '.docNodeKindIcon', '.header-anchor', 'a > svg', + 'nav[aria-label="Breadcrumb"]', + 'nav[aria-label="Previous and next page"]').remove + + css('details > summary').each do |node| + node.parent.remove if node.content.strip == 'On this page' + end + + css('h1, h2, h3, h4, h5, h6').each do |node| + node.css('a.anchor[aria-label="Anchor"]').remove + end css('pre > code').each do |node| - if node['class'] - lang = node['class'][/language-(\w+)/, 1] - node.parent['data-language'] = lang if lang - end - node.parent['data-language'] ||= 'typescript' + language = node['class'].to_s[/\blanguage-([\w-]+)/, 1] + node.parent['data-language'] = language if language end doc diff --git a/test/files/deno_api.html b/test/files/deno_api.html new file mode 100644 index 0000000000..11dd4f603f --- /dev/null +++ b/test/files/deno_api.html @@ -0,0 +1,29 @@ + + + + +
+
+
+
+
  • Deno
  • Network
+
+

Network

+

Networking APIs.

+
+
f
+ Deno.connect +
+
+ 
await Deno.connect();
+ +
+
+
+
+
Did you find what you needed?
+
+
+
Site footer
+ + diff --git a/test/files/deno_runtime.html b/test/files/deno_runtime.html new file mode 100644 index 0000000000..96bec01d0e --- /dev/null +++ b/test/files/deno_runtime.html @@ -0,0 +1,36 @@ + + + +
+
+
+
On this pageOverview
+
+
+ +
+
+

@std/fmt

+

#Overview

+

Provides utilities for formatting text.

+
+ 
console.log('formatted');
+ +
+

Runtime compatibility

+

Works with Deno.

+

Add to your project

+ 
deno add jsr:@std/fmt
+ 
{"imports": {}}
+ See all symbols +
+ +
+
+

Did you find what you needed?

+ Edit this page +
+
+
+ + diff --git a/test/lib/docs/scrapers/deno_test.rb b/test/lib/docs/scrapers/deno_test.rb index c497f1813e..e32bd36522 100644 --- a/test/lib/docs/scrapers/deno_test.rb +++ b/test/lib/docs/scrapers/deno_test.rb @@ -1,4 +1,5 @@ -require_relative '../../test_helper' +require_relative '../../../test_helper' +require_relative '../../../../lib/docs' class DenoScraperTest < Minitest::Test def setup @@ -82,3 +83,74 @@ def test_inherits_from_url_scraper assert @scraper_class < Docs::UrlScraper end end + +class DenoEntriesFilterTest < Minitest::Test + def test_extracts_api_entry_from_content_heading + entry = filter_fixture('deno_api.html', 'api/deno/network/').first + + assert_equal 'Network', entry.name + assert_equal 'API', entry.type + end + + def test_extracts_runtime_entry_from_content_heading + entry = filter_fixture('deno_runtime.html', 'runtime/reference/std/fmt/').first + + assert_equal '@std/fmt', entry.name + assert_equal 'Runtime', entry.type + end + + private + + def filter_fixture(name, path) + html = File.read(File.join(DenoCleanHtmlFilterTest::FIXTURES_PATH, name)) + doc = Docs::Parser.new(html).html + context = { + base_url: Docs::URL.parse('https://docs.deno.com/'), + url: Docs::URL.parse("https://docs.deno.com/#{path}"), + root_path: 'api/', + } + result = { path: path } + content = Docs::Deno::CleanHtmlFilter.new(doc, context, result).call + Docs::Deno::EntriesFilter.new(content, context, result).call + result[:entries] + end +end + +class DenoCleanHtmlFilterTest < Minitest::Test + FIXTURES_PATH = File.expand_path('../../../files', __dir__) + + def test_cleans_api_page_chrome_and_preserves_content + output = filter_fixture('deno_api.html') + + assert_equal 'article', output.name + assert_equal 'Network', output.at_css('h1').content + assert_includes output.text, 'Deno.connect' + refute_includes output.text, 'Site navigation' + refute_includes output.text, 'Did you find what you needed?' + assert_empty output.css('.breadcrumbs, .docNodeKindIcon, .copyButton') + assert_equal 'ts', output.at_css('pre')['data-language'] + end + + def test_cleans_runtime_page_chrome_and_preserves_content + output = filter_fixture('deno_runtime.html') + + assert_equal '@std/fmt', output.at_css('h1').content + assert_includes output.text, 'Runtime compatibility' + assert_includes output.text, 'Add to your project' + assert_includes output.text, 'See all symbols' + refute_includes output.text, 'On this page' + refute_includes output.text, 'Copy page' + refute_includes output.text, 'Did you find what you needed?' + refute_includes output.text, 'Edit this page' + assert_empty output.css('.copyButton, a.anchor, nav') + assert_equal %w(js sh jsonc), output.css('pre').map { |node| node['data-language'] } + end + + private + + def filter_fixture(name) + html = File.read(File.join(FIXTURES_PATH, name)) + doc = Docs::Parser.new(html).html + Docs::Deno::CleanHtmlFilter.new(doc, {}, {}).call + end +end From 99995356e5587b3a995ddba0a3a7b1414986e50f Mon Sep 17 00:00:00 2001 From: Simon Legner Date: Sun, 13 Sep 2026 14:50:45 +0200 Subject: [PATCH 4/4] Update Deno documentation (2.9.6) --- lib/docs/filters/deno/clean_html.rb | 2 +- lib/docs/filters/deno/entries.rb | 29 +++++ lib/docs/scrapers/deno.rb | 170 ++++------------------------ test/files/deno_api.html | 48 ++++++-- test/lib/docs/scrapers/deno_test.rb | 106 ++++++++++++++--- 5 files changed, 177 insertions(+), 178 deletions(-) diff --git a/lib/docs/filters/deno/clean_html.rb b/lib/docs/filters/deno/clean_html.rb index 98df5b2408..7821a5469d 100644 --- a/lib/docs/filters/deno/clean_html.rb +++ b/lib/docs/filters/deno/clean_html.rb @@ -4,7 +4,7 @@ class CleanHtmlFilter < Filter def call @doc = at_css('main#content article', 'article') || doc - css('.breadcrumbs', '.copy-page-split', '.copyButton', + css('.anchor-link', '.breadcrumbs', '.copy-page-split', '.copyButton', '.docNodeKindIcon', '.header-anchor', 'a > svg', 'nav[aria-label="Breadcrumb"]', 'nav[aria-label="Previous and next page"]').remove diff --git a/lib/docs/filters/deno/entries.rb b/lib/docs/filters/deno/entries.rb index 29137c5fbb..209aad45f8 100644 --- a/lib/docs/filters/deno/entries.rb +++ b/lib/docs/filters/deno/entries.rb @@ -14,6 +14,35 @@ def get_name def get_type TYPES_BY_PATH[slug.split('/').first] || 'Guide' end + + # Multi-symbol API pages open with a `#symbol-index` listing every symbol + # that is then documented in full further down the same page. Index those + # symbols individually, then drop the listing: once each symbol has its + # own entry linking to its section, the listing just repeats the page. + def additional_entries + index = at_css('#symbol-index') + return [] unless index + + entries = index.css('.namespaceItem > .namespaceItemContent > a[href*="#"]') + .each_with_object([]) do |link, acc| + target, fragment = link['href'].split('#', 2) + next if fragment.blank? + # Symbols re-exported from another module link to the page that + # documents them; let that page own the entry instead of repeating it. + next unless target.blank? || target == File.basename(path) + symbol = link.content.strip + next if symbol.empty? || acc.any? { |entry| entry.first == symbol } + acc << [symbol, fragment] + end + + index.remove + # The same symbol name is documented by several modules (`Socket` by + # net, dgram and process); qualify each one with the page it belongs to. + # For `node:` modules the page title is the import name, so + # `Profiler.CoverageRange` becomes `Profiler.CoverageRange (inspector)`. + entries.map! { |symbol, fragment| ["#{symbol} (#{name})", fragment] } if name.present? + entries + end end end end diff --git a/lib/docs/scrapers/deno.rb b/lib/docs/scrapers/deno.rb index 055fb87ab9..3230ade10b 100644 --- a/lib/docs/scrapers/deno.rb +++ b/lib/docs/scrapers/deno.rb @@ -1,15 +1,14 @@ module Docs class Deno < UrlScraper self.name = 'Deno' - self.type = 'deno' + self.type = 'simple' self.base_url = 'https://docs.deno.com/' - self.root_path = 'api/' + self.root_path = 'api' self.initial_paths = %w( - api/ - api/deno/ - runtime/ - runtime/fundamentals/ - runtime/reference/ + api/deno + runtime + runtime/fundamentals + runtime/reference ) self.links = { home: 'https://deno.com/', @@ -29,7 +28,19 @@ class Deno < UrlScraper /\Ablog\//, /\Adeploy\//, /\Asubhosting\//, + # Aggregate listings that repeat every symbol already documented on the + # individual module pages. + /\Aapi\/\w+\/all_symbols/, + # Per-symbol URLs now 301 to a fragment of the module page + # (api/web/~/Blob -> api/web/file/#Blob). Because the stored path comes + # from the requested URL, crawling them would file a second copy of the + # whole module page under every symbol it documents. + /\Aapi\/\w+\/~\//, ] + # docs.deno.com links to both `api/node/buffer` and `api/node/buffer/`; + # without this each page is crawled twice, once as `…/buffer` and once as + # `…/buffer/index`. + options[:trailing_slash] = false options[:attribution] = <<-HTML © 2018–2025 the Deno authors
@@ -39,7 +50,7 @@ class Deno < UrlScraper # ── Versions ────────────────────────────────────────────────────── version '2' do - self.release = '2.3.1' + self.release = '2.9.6' end version '1' do @@ -52,148 +63,5 @@ class Deno < UrlScraper def get_latest_version(opts) get_latest_github_release('denoland', 'deno', opts) end - - private - - # ── Module categorisation ───────────────────────────────────────── - - MODULE_CATEGORIES = { - 'Deno' => %w[Deno], - 'Web APIs' => %w[fetch Request Response Headers URL URLSearchParams - FormData Blob File ReadableStream WritableStream - TransformStream TextEncoder TextDecoder - WebSocket EventSource AbortController AbortSignal - crypto CryptoKey SubtleCrypto], - 'I/O' => %w[open read write close seek], - 'File System' => %w[readFile writeFile readDir mkdir remove rename - stat lstat realPath readLink symlink link - truncate copyFile chmod chown], - 'Network' => %w[listen connect serve serveHttp - listenTls connectTls], - 'Subprocess' => %w[run Command ChildProcess], - 'Testing' => %w[test bench], - 'Permissions' => %w[permissions], - }.freeze - - def categorize_module(name) - MODULE_CATEGORIES.each do |category, modules| - return category if modules.any? { |m| name.include?(m) } - end - 'Other' - end - - # ── Page parsing ────────────────────────────────────────────────── - - def parse_page(response) - doc = Nokogiri::HTML.parse(response.body) - return nil if doc.at_css('meta[http-equiv="refresh"]') - - content = doc.at_css('main, article, [role="main"], .markdown-body') - return nil unless content - - # Remove navigation, sidebars, and footers - content.css('nav, footer, .sidebar, .breadcrumb, .toc, - .page-nav, .edit-link, .header-anchor').each(&:remove) - - # Remove script and style tags - content.css('script, style').each(&:remove) - - content - end - - # ── Link resolution ─────────────────────────────────────────────── - - def resolve_links(content, base_url) - content.css('a[href]').each do |link| - href = link['href'] - next if href.nil? || href.empty? - next if href.start_with?('#') - next if href.match?(%r{\Ahttps?://}) && !href.start_with?(self.class.base_url) - - begin - absolute = URI.join(base_url, href).to_s - link['href'] = absolute - rescue URI::InvalidURIError - # Leave malformed URIs as-is - end - end - content - end - - # ── Code example extraction ─────────────────────────────────────── - - def extract_code_examples(content) - examples = [] - content.css('pre > code, pre.highlight, .code-block').each_with_index do |block, idx| - lang = detect_language(block) - source = block.text.strip - next if source.empty? - - examples << { - index: idx, - language: lang, - source: source, - lines: source.lines.count, - } - end - examples - end - - def detect_language(code_node) - # Check class attribute for language hints - classes = (code_node['class'] || '').split - lang_class = classes.find { |c| c.start_with?('language-', 'lang-', 'highlight-') } - if lang_class - return lang_class.sub(/\A(?:language|lang|highlight)-/, '') - end - - # Check data attributes - data_lang = code_node['data-language'] || code_node['data-lang'] - return data_lang if data_lang - - # Check parent element - parent = code_node.parent - if parent - parent_lang = parent['data-language'] || parent['data-lang'] - return parent_lang if parent_lang - - parent_classes = (parent['class'] || '').split - parent_lang_class = parent_classes.find { |c| c.start_with?('language-', 'lang-') } - if parent_lang_class - return parent_lang_class.sub(/\A(?:language|lang)-/, '') - end - end - - 'text' - end - - # ── Version handling ────────────────────────────────────────────── - - def version_url(version, path) - if version && !version.empty? - "#{self.class.base_url}#{path}@#{version}" - else - "#{self.class.base_url}#{path}" - end - end - - def parse_version_from_url(url) - match = url.match(/@([\d.]+)/) - match ? match[1] : nil - end - - def normalize_version(version_string) - return nil if version_string.nil? || version_string.empty? - - # Strip leading 'v' if present - cleaned = version_string.sub(/\Av/, '') - - # Validate semver-like format - parts = cleaned.split('.') - return nil unless parts.length.between?(1, 3) - return nil unless parts.all? { |p| p.match?(/\A\d+\z/) } - - cleaned - end end end diff --git a/test/files/deno_api.html b/test/files/deno_api.html index 11dd4f603f..55dea0770d 100644 --- a/test/files/deno_api.html +++ b/test/files/deno_api.html @@ -5,19 +5,49 @@
-
+
  • Deno
  • Network

Network

Networking APIs.

-
-
f
- Deno.connect -
-
- 
await Deno.connect();
- -
+
+
+

Functions

+
+
+
f
+
+ Deno.connect +
Connects to a TCP listener.
+ +
+
+
+
+
+

Classes

+
+
+
c
+
+ Blob +
Re-exported from another module.
+
+
+
+
+
+
+

Deno.connect #

+

Parameters #

+ options#: ConnectOptions +
+ 
await Deno.connect();
+ +
+
diff --git a/test/lib/docs/scrapers/deno_test.rb b/test/lib/docs/scrapers/deno_test.rb index e32bd36522..b24b90c46f 100644 --- a/test/lib/docs/scrapers/deno_test.rb +++ b/test/lib/docs/scrapers/deno_test.rb @@ -11,7 +11,7 @@ def test_scraper_name end def test_scraper_type - assert_equal 'deno', @scraper_class.type + assert_equal 'simple', @scraper_class.type end def test_base_url @@ -19,14 +19,14 @@ def test_base_url end def test_root_path - assert_equal 'api/', @scraper_class.root_path + assert_equal 'api', @scraper_class.root_path end def test_initial_paths_present assert_kind_of Array, @scraper_class.initial_paths refute_empty @scraper_class.initial_paths - assert_includes @scraper_class.initial_paths, 'api/' - assert_includes @scraper_class.initial_paths, 'runtime/' + assert_includes @scraper_class.initial_paths, 'api/deno' + assert_includes @scraper_class.initial_paths, 'runtime' end def test_links_defined @@ -56,6 +56,29 @@ def test_skip_patterns_excludes_deploy assert patterns.any? { |p| 'deploy/docs' =~ p } end + # Aggregate pages repeat every symbol documented on the module pages. + def test_skip_patterns_excludes_all_symbols_pages + patterns = @scraper_class.options[:skip_patterns] + %w(api/node/all_symbols api/deno/all_symbols api/web/all_symbols).each do |path| + assert patterns.any? { |p| path =~ p }, "expected #{path} to be skipped" + end + end + + # These 301 to a fragment of the module page, so crawling them would file a + # duplicate copy of that page under every symbol it documents. + def test_skip_patterns_excludes_per_symbol_redirect_urls + patterns = @scraper_class.options[:skip_patterns] + %w(api/web/~/Blob api/deno/~/Deno.connect api/node/~/SlowBuffer).each do |path| + assert patterns.any? { |p| path =~ p }, "expected #{path} to be skipped" + end + end + + # Without this the site's two link forms (`api/node/buffer` and + # `api/node/buffer/`) are crawled as two pages with identical content. + def test_trailing_slashes_are_normalized_away + assert_equal false, @scraper_class.options[:trailing_slash] + end + def test_attribution_present attribution = @scraper_class.options[:attribution] assert_kind_of String, attribution @@ -69,16 +92,6 @@ def test_has_versions refute_empty versions end - def test_module_categories_frozen - categories = Docs::Deno::MODULE_CATEGORIES - assert categories.frozen? - assert_kind_of Hash, categories - assert categories.key?('Deno') - assert categories.key?('Web APIs') - assert categories.key?('File System') - assert categories.key?('Network') - end - def test_inherits_from_url_scraper assert @scraper_class < Docs::UrlScraper end @@ -86,14 +99,69 @@ def test_inherits_from_url_scraper class DenoEntriesFilterTest < Minitest::Test def test_extracts_api_entry_from_content_heading - entry = filter_fixture('deno_api.html', 'api/deno/network/').first + entry = filter_fixture('deno_api.html', 'api/deno/network').first assert_equal 'Network', entry.name assert_equal 'API', entry.type end + def test_extracts_symbol_entries_from_multi_symbol_api_page + entries = filter_fixture('deno_api.html', 'api/deno/network') + symbol = entries.find { |entry| entry.name == 'Deno.connect (Network)' } + + refute_nil symbol + assert_equal 'api/deno/network#Deno.connect', symbol.path + assert_equal 'API', symbol.type + end + + # `Socket` is documented by net, dgram and process; the module name is what + # tells the three entries apart. For node: modules it is the import name. + def test_qualifies_symbol_entries_with_the_module_name + names = filter_fixture('deno_api.html', 'api/deno/network').map(&:name) + + assert_includes names, 'Deno.connect (Network)' + refute_includes names, 'Deno.connect' + end + + # The listing repeats every symbol documented in full below it. + def test_removes_the_symbol_index_after_extracting_entries + html = File.read(File.join(DenoCleanHtmlFilterTest::FIXTURES_PATH, 'deno_api.html')) + doc = Docs::Parser.new(html).html + context = { base_url: Docs::URL.parse('https://docs.deno.com/'), + url: Docs::URL.parse('https://docs.deno.com/api/deno/network'), + root_path: 'api' } + result = { path: 'api/deno/network' } + content = Docs::Deno::CleanHtmlFilter.new(doc, context, result).call + content = Docs::Deno::EntriesFilter.new(content, context, result).call + + assert_nil content.at_css('#symbol-index') + refute_empty content.css('.symbolGroup') + assert_includes content.text, 'Deno.connect' + end + + # Blob is documented on api/web/file; that page owns the entry. + def test_skips_symbols_re_exported_from_another_page + names = filter_fixture('deno_api.html', 'api/deno/network').map(&:name) + + refute_includes names, 'Blob' + refute_includes names, 'Blob (Network)' + end + + def test_skips_namespace_sub_item_links + names = filter_fixture('deno_api.html', 'api/deno/network').map(&:name) + + refute_includes names, 'addr' + refute_includes names, 'addr (Network)' + end + + def test_runtime_page_has_no_symbol_entries + entries = filter_fixture('deno_runtime.html', 'runtime/reference/std/fmt') + + assert_equal 1, entries.size + end + def test_extracts_runtime_entry_from_content_heading - entry = filter_fixture('deno_runtime.html', 'runtime/reference/std/fmt/').first + entry = filter_fixture('deno_runtime.html', 'runtime/reference/std/fmt').first assert_equal '@std/fmt', entry.name assert_equal 'Runtime', entry.type @@ -107,7 +175,7 @@ def filter_fixture(name, path) context = { base_url: Docs::URL.parse('https://docs.deno.com/'), url: Docs::URL.parse("https://docs.deno.com/#{path}"), - root_path: 'api/', + root_path: 'api', } result = { path: path } content = Docs::Deno::CleanHtmlFilter.new(doc, context, result).call @@ -128,6 +196,10 @@ def test_cleans_api_page_chrome_and_preserves_content refute_includes output.text, 'Site navigation' refute_includes output.text, 'Did you find what you needed?' assert_empty output.css('.breadcrumbs, .docNodeKindIcon, .copyButton') + assert_empty output.css('.anchor-link') + # The markers sit inside headings and code, where a stray "#" corrupts the text. + assert_equal 'Deno.connect', output.at_css('.symbolGroup h2').content.strip + assert_equal 'options: ConnectOptions', output.at_css('code').content.strip assert_equal 'ts', output.at_css('pre')['data-language'] end