diff --git a/crawl4ai/content_filter_strategy.py b/crawl4ai/content_filter_strategy.py index ab99a793c..9a9b2490f 100644 --- a/crawl4ai/content_filter_strategy.py +++ b/crawl4ai/content_filter_strategy.py @@ -580,8 +580,10 @@ def __init__( min_word_threshold (int): Minimum word threshold for filtering (optional). threshold_type (str): Threshold type for dynamic threshold (default: 'fixed'). threshold (float): Fixed threshold value (default: 0.48). - preserve_classes (list): CSS class names to always keep regardless of score (optional). - preserve_tags (list): HTML tag names to always keep regardless of score (optional). + preserve_classes (list): CSS class names to always keep, regardless of score + or of the element being an excluded tag (optional). + preserve_tags (list): HTML tag names to always keep, regardless of score + or of the tag being excluded by default (optional). """ super().__init__(None) self.min_word_threshold = min_word_threshold @@ -683,9 +685,9 @@ def _remove_comments(self, soup): element.extract() def _remove_unwanted_tags(self, soup): - """Removes unwanted tags""" - for tag in self.excluded_tags: - for element in soup.find_all(tag): + """Removes unwanted tags, except the ones on the preserve whitelist""" + for element in soup.find_all(list(self.excluded_tags)): + if not self._is_preserved(element): element.decompose() def _is_preserved(self, node): diff --git a/tests/test_pruning_preserve_whitelist_1900.py b/tests/test_pruning_preserve_whitelist_1900.py index 01a46817b..73954ca10 100644 --- a/tests/test_pruning_preserve_whitelist_1900.py +++ b/tests/test_pruning_preserve_whitelist_1900.py @@ -201,14 +201,20 @@ def test_both_classes_and_tags(self): assert "alice" in combined assert "Apr 6, 2026" in combined - def test_whitelist_does_not_override_excluded_tags(self): - """Nav/footer/header are removed before pruning — whitelist can't save them.""" + def test_whitelist_overrides_excluded_tags(self): + """A whitelisted tag survives even when it is in excluded_tags.""" f = PruningContentFilter(preserve_tags=["nav"]) - result = f.filter_content(GITHUB_COMMENT_HTML) - combined = " ".join(result) - # nav is in excluded_tags and removed before pruning runs - # preserve_tags only affects the pruning phase - # This is expected — excluded_tags are structural boilerplate + combined = " ".join(f.filter_content(GITHUB_COMMENT_HTML)) + assert "About" in combined + # non-whitelisted excluded tags are still dropped + assert "Copyright 2026" not in combined + + def test_whitelisted_class_on_excluded_tag(self): + """A whitelisted class saves an excluded tag too.""" + f = PruningContentFilter(preserve_classes=["site-footer"]) + combined = " ".join(f.filter_content(GITHUB_COMMENT_HTML)) + assert "Copyright 2026" in combined + assert "About" not in combined # ── _is_preserved method ─────────────────────────────────────────────────