From b7fcf6c25fa00027b1cbecd04920d89b91c616c6 Mon Sep 17 00:00:00 2001 From: Battleplus <3559424769@qq.com> Date: Sat, 22 Aug 2026 09:44:32 +0800 Subject: [PATCH 1/2] fix: respect preserve_tags/preserve_classes in _remove_unwanted_tags Previously, _remove_unwanted_tags() decomposed all matching elements regardless of the preserve whitelist. Now it checks _is_preserved() before decomposing, so elements matching preserve_tags or preserve_classes are kept even if their tag is in excluded_tags. Fixes #2125 Signed-off-by: Battleplus <3559424769@qq.com> --- crawl4ai/content_filter_strategy.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crawl4ai/content_filter_strategy.py b/crawl4ai/content_filter_strategy.py index ab99a793c..608431bfc 100644 --- a/crawl4ai/content_filter_strategy.py +++ b/crawl4ai/content_filter_strategy.py @@ -683,10 +683,11 @@ def _remove_comments(self, soup): element.extract() def _remove_unwanted_tags(self, soup): - """Removes unwanted tags""" + """Removes unwanted tags, but respects preserve_tags/preserve_classes whitelist.""" for tag in self.excluded_tags: for element in soup.find_all(tag): - element.decompose() + if not self._is_preserved(element): + element.decompose() def _is_preserved(self, node): """Check if a node matches the preserve whitelist.""" From 5a91fad1e2c653ad9ad9c3616bfdade3fd3bbfcd Mon Sep 17 00:00:00 2001 From: Battleplus <121445871+Battleplus@users.noreply.github.com> Date: Sat, 22 Aug 2026 16:11:12 +0800 Subject: [PATCH 2/2] test: cover preserved excluded tags --- tests/test_pruning_preserve_whitelist_1900.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test_pruning_preserve_whitelist_1900.py b/tests/test_pruning_preserve_whitelist_1900.py index 01a46817b..402f82a50 100644 --- a/tests/test_pruning_preserve_whitelist_1900.py +++ b/tests/test_pruning_preserve_whitelist_1900.py @@ -201,14 +201,20 @@ def test_both_classes_and_tags(self): assert "alice" in combined assert "Apr 6, 2026" in combined - def test_whitelist_does_not_override_excluded_tags(self): - """Nav/footer/header are removed before pruning — whitelist can't save them.""" + def test_preserve_tag_overrides_excluded_tags(self): + """A preserved tag remains even when it is structural boilerplate by default.""" f = PruningContentFilter(preserve_tags=["nav"]) result = f.filter_content(GITHUB_COMMENT_HTML) combined = " ".join(result) - # nav is in excluded_tags and removed before pruning runs - # preserve_tags only affects the pruning phase - # This is expected — excluded_tags are structural boilerplate + assert "Home" in combined + assert "About" in combined + + def test_preserve_class_overrides_excluded_tags(self): + """A preserved class remains even when its element tag is excluded.""" + f = PruningContentFilter(preserve_classes=["site-footer"]) + result = f.filter_content(GITHUB_COMMENT_HTML) + combined = " ".join(result) + assert "Copyright 2026" in combined # ── _is_preserved method ─────────────────────────────────────────────────