perf: make html_to_text linear in the number of nodes

src/crawlee/_utils/html_to_text.py

@@ -31,6 +31,4 @@
     'option',
 }
 
-_EMPTY_OR_ENDS_WITH_ANY_WHITE_SPACE = re.compile(r'(^|\s)$')
-_EMPTY_OR_ENDS_WITH_NEW_LINE = re.compile(r'(^|\n)$')
 _ANY_CONSECUTIVE_WHITE_SPACES = re.compile(r'\s+')

src/crawlee/crawlers/_beautifulsoup/_utils.py

@@ -5,13 +5,7 @@
 
 from bs4 import BeautifulSoup, NavigableString, PageElement, Tag
 
-from crawlee._utils.html_to_text import (
-    _ANY_CONSECUTIVE_WHITE_SPACES,
-    _EMPTY_OR_ENDS_WITH_ANY_WHITE_SPACE,
-    _EMPTY_OR_ENDS_WITH_NEW_LINE,
-    BLOCK_TAGS,
-    SKIP_TAGS,
-)
+from crawlee._utils.html_to_text import _ANY_CONSECUTIVE_WHITE_SPACES, BLOCK_TAGS, SKIP_TAGS
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
@@ -33,15 +27,22 @@ def html_to_text(source: str | Tag) -> str:
     else:
         raise TypeError('Source must be either a string or a `BeautifulSoup` object.')
 
-    text = ''
+    parts: list[str] = []
+    # Last emitted character, so the whitespace checks never have to read the growing output back.
+    last = ''
+
+    def push(part: str) -> None:
+        nonlocal last
+        if part:
+            parts.append(part)
+            last = part[-1]
 
     def _page_element_to_text(page_elements: Iterable[PageElement]) -> None:
         """Extract and process text content from a collection of HTML elements.
 
         Convert page elements into plain text while preserving structure. Handle whitespace compression,
         skip unwanted elements, and format block elements correctly.
         """
-        nonlocal text
         for page_element in page_elements:
             if isinstance(page_element, (Tag, NavigableString)):
                 if isinstance(page_element, NavigableString):
@@ -52,26 +53,26 @@ def _page_element_to_text(page_elements: Iterable[PageElement]) -> None:
                         # Compress white spaces outside of pre block
                         compr = re.sub(_ANY_CONSECUTIVE_WHITE_SPACES, ' ', page_element.get_text())
                     # If text is empty or ends with a whitespace, don't add the leading whitespace or new line
-                    if (compr.startswith((' ', '\n'))) and re.search(_EMPTY_OR_ENDS_WITH_ANY_WHITE_SPACE, text):
+                    if (compr.startswith((' ', '\n'))) and (not last or last.isspace()):
                         compr = compr[1:]
-                    text += compr
+                    push(compr)
                 elif page_element.name.lower() in SKIP_TAGS:
                     # Skip comments and special elements
                     pass
                 elif page_element.name.lower() == 'br':
-                    text += '\n'
+                    push('\n')
                 elif page_element.name.lower() == 'td':
                     _page_element_to_text(page_element.children)
-                    text += '\t'
+                    push('\t')
                 else:
                     # Block elements must be surrounded by newlines(unless beginning of text)
                     is_block_tag = page_element.name.lower() in BLOCK_TAGS
-                    if is_block_tag and not re.search(_EMPTY_OR_ENDS_WITH_NEW_LINE, text):
-                        text += '\n'
+                    if is_block_tag and last and last != '\n':
+                        push('\n')
                     _page_element_to_text(page_element.children)
-                    if is_block_tag and not text.endswith('\n'):
-                        text += '\n'
+                    if is_block_tag and last != '\n':
+                        push('\n')
 
     _page_element_to_text(soup.children)
 
-    return text.strip()
+    return ''.join(parts).strip()

src/crawlee/crawlers/_parsel/_utils.py

@@ -4,13 +4,7 @@
 
 from parsel import Selector
 
-from crawlee._utils.html_to_text import (
-    _ANY_CONSECUTIVE_WHITE_SPACES,
-    _EMPTY_OR_ENDS_WITH_ANY_WHITE_SPACE,
-    _EMPTY_OR_ENDS_WITH_NEW_LINE,
-    BLOCK_TAGS,
-    SKIP_TAGS,
-)
+from crawlee._utils.html_to_text import _ANY_CONSECUTIVE_WHITE_SPACES, BLOCK_TAGS, SKIP_TAGS
 
 
 def html_to_text(source: str | Selector) -> str:
@@ -29,7 +23,15 @@ def html_to_text(source: str | Selector) -> str:
     else:
         raise TypeError('Source must be either a string or a `Selector` object.')
 
-    text = ''
+    parts: list[str] = []
+    # Last emitted character, so the whitespace checks never have to read the growing output back.
+    last = ''
+
+    def push(part: str) -> None:
+        nonlocal last
+        if part:
+            parts.append(part)
+            last = part[-1]
 
     def _extract_text(elements: list[Selector], *, compress: bool = True) -> None:
         """Extract text content from HTML elements while preserving formatting.
@@ -41,38 +43,37 @@ def _extract_text(elements: list[Selector], *, compress: bool = True) -> None:
             elements: A list of selectors representing the HTML elements.
             compress: Whether to compress consecutive whitespace outside of `<pre>` blocks.
         """
-        nonlocal text
         for element in elements:
             tag = element.root.tag if hasattr(element.root, 'tag') else None
 
             if tag is None:
                 # Compress white spaces outside of pre block
                 compr = re.sub(_ANY_CONSECUTIVE_WHITE_SPACES, ' ', element.root) if compress else element.root
                 # If text is empty or ends with a whitespace, don't add the leading whitespace or new line
-                if (compr.startswith((' ', '\n'))) and re.search(_EMPTY_OR_ENDS_WITH_ANY_WHITE_SPACE, text):
+                if (compr.startswith((' ', '\n'))) and (not last or last.isspace()):
                     compr = compr[1:]
-                text += compr
+                push(compr)
 
             if tag in SKIP_TAGS or not isinstance(tag, str):
                 continue
 
             if tag == 'br':
-                text += '\n'
+                push('\n')
             elif tag == 'td':
                 _extract_text(element.xpath('./node()'))
-                text += '\t'
+                push('\t')
             else:
                 is_block_tag = tag in BLOCK_TAGS if tag else False
 
-                if is_block_tag and not re.search(_EMPTY_OR_ENDS_WITH_NEW_LINE, text):
-                    text += '\n'
+                if is_block_tag and last and last != '\n':
+                    push('\n')
 
                 _extract_text(element.xpath('./node()'), compress=tag != 'pre')
 
-                if is_block_tag and not text.endswith('\n'):
-                    text += '\n'
+                if is_block_tag and last != '\n':
+                    push('\n')
 
     # Start processing the root elements
     _extract_text(selector.xpath('/*'))
 
-    return text.strip()
+    return ''.join(parts).strip()

tests/unit/_utils/test_html_to_text.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import time
 from typing import TYPE_CHECKING
 
 import pytest
@@ -201,3 +202,23 @@ def test_html_to_text_parsel() -> None:
 
 def test_html_to_text_beautifulsoup() -> None:
     assert html_to_text_beautifulsoup(BeautifulSoup(_EXAMPLE_HTML, features='lxml')) == _EXPECTED_TEXT
+
+
+@pytest.mark.parametrize('html_to_text', [html_to_text_parsel, html_to_text_beautifulsoup])
+def test_html_to_text_is_linear_in_node_count(html_to_text: Callable[[str], str]) -> None:
+    """Converting four times as many sibling nodes takes about four times as long, not sixteen times."""
+
+    def convert(repeats: int) -> float:
+        source = '<html><body><div>' + '<i> a</i>' * repeats + '</div></body></html>'
+        started = time.perf_counter()
+        text = html_to_text(source)
+        elapsed = time.perf_counter() - started
+        assert text == ' '.join(['a'] * repeats)
+        return elapsed
+
+    # Warm up the parser so one-off setup costs do not skew the small run.
+    convert(1000)
+    small = convert(20_000)
+    large = convert(80_000)
+
+    assert large / small < 10