diff --git a/src/crawlee/_utils/html_to_text.py b/src/crawlee/_utils/html_to_text.py
index 804b55f464..0558de2fed 100644
--- a/src/crawlee/_utils/html_to_text.py
+++ b/src/crawlee/_utils/html_to_text.py
@@ -31,6 +31,4 @@
'option',
}
-_EMPTY_OR_ENDS_WITH_ANY_WHITE_SPACE = re.compile(r'(^|\s)$')
-_EMPTY_OR_ENDS_WITH_NEW_LINE = re.compile(r'(^|\n)$')
_ANY_CONSECUTIVE_WHITE_SPACES = re.compile(r'\s+')
diff --git a/src/crawlee/crawlers/_beautifulsoup/_utils.py b/src/crawlee/crawlers/_beautifulsoup/_utils.py
index 53922f84ab..afaf57200c 100644
--- a/src/crawlee/crawlers/_beautifulsoup/_utils.py
+++ b/src/crawlee/crawlers/_beautifulsoup/_utils.py
@@ -5,13 +5,7 @@
from bs4 import BeautifulSoup, NavigableString, PageElement, Tag
-from crawlee._utils.html_to_text import (
- _ANY_CONSECUTIVE_WHITE_SPACES,
- _EMPTY_OR_ENDS_WITH_ANY_WHITE_SPACE,
- _EMPTY_OR_ENDS_WITH_NEW_LINE,
- BLOCK_TAGS,
- SKIP_TAGS,
-)
+from crawlee._utils.html_to_text import _ANY_CONSECUTIVE_WHITE_SPACES, BLOCK_TAGS, SKIP_TAGS
if TYPE_CHECKING:
from collections.abc import Iterable
@@ -33,7 +27,15 @@ def html_to_text(source: str | Tag) -> str:
else:
raise TypeError('Source must be either a string or a `BeautifulSoup` object.')
- text = ''
+ parts: list[str] = []
+ # Last emitted character, so the whitespace checks never have to read the growing output back.
+ last = ''
+
+ def push(part: str) -> None:
+ nonlocal last
+ if part:
+ parts.append(part)
+ last = part[-1]
def _page_element_to_text(page_elements: Iterable[PageElement]) -> None:
"""Extract and process text content from a collection of HTML elements.
@@ -41,7 +43,6 @@ def _page_element_to_text(page_elements: Iterable[PageElement]) -> None:
Convert page elements into plain text while preserving structure. Handle whitespace compression,
skip unwanted elements, and format block elements correctly.
"""
- nonlocal text
for page_element in page_elements:
if isinstance(page_element, (Tag, NavigableString)):
if isinstance(page_element, NavigableString):
@@ -52,26 +53,26 @@ def _page_element_to_text(page_elements: Iterable[PageElement]) -> None:
# Compress white spaces outside of pre block
compr = re.sub(_ANY_CONSECUTIVE_WHITE_SPACES, ' ', page_element.get_text())
# If text is empty or ends with a whitespace, don't add the leading whitespace or new line
- if (compr.startswith((' ', '\n'))) and re.search(_EMPTY_OR_ENDS_WITH_ANY_WHITE_SPACE, text):
+ if (compr.startswith((' ', '\n'))) and (not last or last.isspace()):
compr = compr[1:]
- text += compr
+ push(compr)
elif page_element.name.lower() in SKIP_TAGS:
# Skip comments and special elements
pass
elif page_element.name.lower() == 'br':
- text += '\n'
+ push('\n')
elif page_element.name.lower() == 'td':
_page_element_to_text(page_element.children)
- text += '\t'
+ push('\t')
else:
# Block elements must be surrounded by newlines(unless beginning of text)
is_block_tag = page_element.name.lower() in BLOCK_TAGS
- if is_block_tag and not re.search(_EMPTY_OR_ENDS_WITH_NEW_LINE, text):
- text += '\n'
+ if is_block_tag and last and last != '\n':
+ push('\n')
_page_element_to_text(page_element.children)
- if is_block_tag and not text.endswith('\n'):
- text += '\n'
+ if is_block_tag and last != '\n':
+ push('\n')
_page_element_to_text(soup.children)
- return text.strip()
+ return ''.join(parts).strip()
diff --git a/src/crawlee/crawlers/_parsel/_utils.py b/src/crawlee/crawlers/_parsel/_utils.py
index 19706b7564..7ca143d5ff 100644
--- a/src/crawlee/crawlers/_parsel/_utils.py
+++ b/src/crawlee/crawlers/_parsel/_utils.py
@@ -4,13 +4,7 @@
from parsel import Selector
-from crawlee._utils.html_to_text import (
- _ANY_CONSECUTIVE_WHITE_SPACES,
- _EMPTY_OR_ENDS_WITH_ANY_WHITE_SPACE,
- _EMPTY_OR_ENDS_WITH_NEW_LINE,
- BLOCK_TAGS,
- SKIP_TAGS,
-)
+from crawlee._utils.html_to_text import _ANY_CONSECUTIVE_WHITE_SPACES, BLOCK_TAGS, SKIP_TAGS
def html_to_text(source: str | Selector) -> str:
@@ -29,7 +23,15 @@ def html_to_text(source: str | Selector) -> str:
else:
raise TypeError('Source must be either a string or a `Selector` object.')
- text = ''
+ parts: list[str] = []
+ # Last emitted character, so the whitespace checks never have to read the growing output back.
+ last = ''
+
+ def push(part: str) -> None:
+ nonlocal last
+ if part:
+ parts.append(part)
+ last = part[-1]
def _extract_text(elements: list[Selector], *, compress: bool = True) -> None:
"""Extract text content from HTML elements while preserving formatting.
@@ -41,7 +43,6 @@ def _extract_text(elements: list[Selector], *, compress: bool = True) -> None:
elements: A list of selectors representing the HTML elements.
compress: Whether to compress consecutive whitespace outside of `` blocks.
"""
- nonlocal text
for element in elements:
tag = element.root.tag if hasattr(element.root, 'tag') else None
@@ -49,30 +50,30 @@ def _extract_text(elements: list[Selector], *, compress: bool = True) -> None:
# Compress white spaces outside of pre block
compr = re.sub(_ANY_CONSECUTIVE_WHITE_SPACES, ' ', element.root) if compress else element.root
# If text is empty or ends with a whitespace, don't add the leading whitespace or new line
- if (compr.startswith((' ', '\n'))) and re.search(_EMPTY_OR_ENDS_WITH_ANY_WHITE_SPACE, text):
+ if (compr.startswith((' ', '\n'))) and (not last or last.isspace()):
compr = compr[1:]
- text += compr
+ push(compr)
if tag in SKIP_TAGS or not isinstance(tag, str):
continue
if tag == 'br':
- text += '\n'
+ push('\n')
elif tag == 'td':
_extract_text(element.xpath('./node()'))
- text += '\t'
+ push('\t')
else:
is_block_tag = tag in BLOCK_TAGS if tag else False
- if is_block_tag and not re.search(_EMPTY_OR_ENDS_WITH_NEW_LINE, text):
- text += '\n'
+ if is_block_tag and last and last != '\n':
+ push('\n')
_extract_text(element.xpath('./node()'), compress=tag != 'pre')
- if is_block_tag and not text.endswith('\n'):
- text += '\n'
+ if is_block_tag and last != '\n':
+ push('\n')
# Start processing the root elements
_extract_text(selector.xpath('/*'))
- return text.strip()
+ return ''.join(parts).strip()
diff --git a/tests/unit/_utils/test_html_to_text.py b/tests/unit/_utils/test_html_to_text.py
index f49feb5b9a..ccaf7c2479 100644
--- a/tests/unit/_utils/test_html_to_text.py
+++ b/tests/unit/_utils/test_html_to_text.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import time
from typing import TYPE_CHECKING
import pytest
@@ -201,3 +202,23 @@ def test_html_to_text_parsel() -> None:
def test_html_to_text_beautifulsoup() -> None:
assert html_to_text_beautifulsoup(BeautifulSoup(_EXAMPLE_HTML, features='lxml')) == _EXPECTED_TEXT
+
+
+@pytest.mark.parametrize('html_to_text', [html_to_text_parsel, html_to_text_beautifulsoup])
+def test_html_to_text_is_linear_in_node_count(html_to_text: Callable[[str], str]) -> None:
+ """Converting four times as many sibling nodes takes about four times as long, not sixteen times."""
+
+ def convert(repeats: int) -> float:
+ source = '' + ' a' * repeats + '
'
+ started = time.perf_counter()
+ text = html_to_text(source)
+ elapsed = time.perf_counter() - started
+ assert text == ' '.join(['a'] * repeats)
+ return elapsed
+
+ # Warm up the parser so one-off setup costs do not skew the small run.
+ convert(1000)
+ small = convert(20_000)
+ large = convert(80_000)
+
+ assert large / small < 10