diff --git a/crawl4ai/content_filter_strategy.py b/crawl4ai/content_filter_strategy.py index ab99a793c..608431bfc 100644 --- a/crawl4ai/content_filter_strategy.py +++ b/crawl4ai/content_filter_strategy.py @@ -683,10 +683,11 @@ def _remove_comments(self, soup): element.extract() def _remove_unwanted_tags(self, soup): - """Removes unwanted tags""" + """Removes unwanted tags, but respects preserve_tags/preserve_classes whitelist.""" for tag in self.excluded_tags: for element in soup.find_all(tag): - element.decompose() + if not self._is_preserved(element): + element.decompose() def _is_preserved(self, node): """Check if a node matches the preserve whitelist.""" diff --git a/tests/test_pruning_preserve_whitelist_1900.py b/tests/test_pruning_preserve_whitelist_1900.py index 01a46817b..402f82a50 100644 --- a/tests/test_pruning_preserve_whitelist_1900.py +++ b/tests/test_pruning_preserve_whitelist_1900.py @@ -201,14 +201,20 @@ def test_both_classes_and_tags(self): assert "alice" in combined assert "Apr 6, 2026" in combined - def test_whitelist_does_not_override_excluded_tags(self): - """Nav/footer/header are removed before pruning — whitelist can't save them.""" + def test_preserve_tag_overrides_excluded_tags(self): + """A preserved tag remains even when it is structural boilerplate by default.""" f = PruningContentFilter(preserve_tags=["nav"]) result = f.filter_content(GITHUB_COMMENT_HTML) combined = " ".join(result) - # nav is in excluded_tags and removed before pruning runs - # preserve_tags only affects the pruning phase - # This is expected — excluded_tags are structural boilerplate + assert "Home" in combined + assert "About" in combined + + def test_preserve_class_overrides_excluded_tags(self): + """A preserved class remains even when its element tag is excluded.""" + f = PruningContentFilter(preserve_classes=["site-footer"]) + result = f.filter_content(GITHUB_COMMENT_HTML) + combined = " ".join(result) + assert "Copyright 2026" in combined # ── _is_preserved method ─────────────────────────────────────────────────