fix(content-filter): honor preserve whitelist for excluded tags

crawl4ai/content_filter_strategy.py

@@ -580,8 +580,10 @@ def __init__(
             min_word_threshold (int): Minimum word threshold for filtering (optional).
             threshold_type (str): Threshold type for dynamic threshold (default: 'fixed').
             threshold (float): Fixed threshold value (default: 0.48).
-            preserve_classes (list): CSS class names to always keep regardless of score (optional).
-            preserve_tags (list): HTML tag names to always keep regardless of score (optional).
+            preserve_classes (list): CSS class names to always keep, regardless of score
+                or of the element being an excluded tag (optional).
+            preserve_tags (list): HTML tag names to always keep, regardless of score
+                or of the tag being excluded by default (optional).
         """
         super().__init__(None)
         self.min_word_threshold = min_word_threshold
@@ -683,9 +685,9 @@ def _remove_comments(self, soup):
             element.extract()
 
     def _remove_unwanted_tags(self, soup):
-        """Removes unwanted tags"""
-        for tag in self.excluded_tags:
-            for element in soup.find_all(tag):
+        """Removes unwanted tags, except the ones on the preserve whitelist"""
+        for element in soup.find_all(list(self.excluded_tags)):
+            if not self._is_preserved(element):
                 element.decompose()
 
     def _is_preserved(self, node):

tests/test_pruning_preserve_whitelist_1900.py

@@ -201,14 +201,20 @@ def test_both_classes_and_tags(self):
         assert "alice" in combined
         assert "Apr 6, 2026" in combined
 
-    def test_whitelist_does_not_override_excluded_tags(self):
-        """Nav/footer/header are removed before pruning — whitelist can't save them."""
+    def test_whitelist_overrides_excluded_tags(self):
+        """A whitelisted tag survives even when it is in excluded_tags."""
         f = PruningContentFilter(preserve_tags=["nav"])
-        result = f.filter_content(GITHUB_COMMENT_HTML)
-        combined = " ".join(result)
-        # nav is in excluded_tags and removed before pruning runs
-        # preserve_tags only affects the pruning phase
-        # This is expected — excluded_tags are structural boilerplate
+        combined = " ".join(f.filter_content(GITHUB_COMMENT_HTML))
+        assert "About" in combined
+        # non-whitelisted excluded tags are still dropped
+        assert "Copyright 2026" not in combined
+
+    def test_whitelisted_class_on_excluded_tag(self):
+        """A whitelisted class saves an excluded tag too."""
+        f = PruningContentFilter(preserve_classes=["site-footer"])
+        combined = " ".join(f.filter_content(GITHUB_COMMENT_HTML))
+        assert "Copyright 2026" in combined
+        assert "About" not in combined
 
 
 # ── _is_preserved method ─────────────────────────────────────────────────