probe: recognise an empty SPA shell, send a SPA to its API before its sitemap

accesspath/probe.py

@@ -756,6 +756,16 @@ async def _batch(entries: list[tuple[str, str | None, str]]) -> list:
 
 
 # ── Tier 2 : analyse HTML statique ─────────────────────────────────────────
+SPA_MAX_WORDS = 50
+SPA_EMPTY_ROOT = re.compile(r'<(app-root|div id="(?:root|app|__next|__nuxt)")[^>]*>\s*</', re.I)
+
+
+def visible_words(html: str) -> int:
+    """Le nombre de mots visibles : le texte hors des scripts, des styles et des balises."""
+    t = re.sub(r"<(script|style|noscript|template)\b[^>]*>.*?</\1>", " ", html, flags=re.S | re.I)
+    return len(re.sub(r"<[^>]+>", " ", t).split())
+
+
 async def discover_tier2(url: str, *, home: dict | None = None) -> dict:
     """Fetch l'URL principale et analyse :
     - framework signatures (WP, Next.js __NEXT_DATA__, Apollo, Hotwire/Turbo)
@@ -788,7 +798,8 @@ async def discover_tier2(url: str, *, home: dict | None = None) -> dict:
         out["frameworks"].append("rails_turbo")
     if "data-react-helmet" in html or "react-root" in html or 'id="root"' in html:
         out["frameworks"].append("react")
-    if "ng-version" in html or "ng-app" in html:
+    # Angular CLI récent : ni ng-version ni ng-app dans le HTML servi, seulement l'élément racine <app-root>.
+    if "ng-version" in html or "ng-app" in html or "<app-root" in html:
         out["frameworks"].append("angular")
     if "data-vue" in html or "v-cloak" in html:
         out["frameworks"].append("vue")
@@ -820,7 +831,12 @@ async def discover_tier2(url: str, *, home: dict | None = None) -> dict:
     # SPA heuristic : peu de patterns "data" + framework JS détecté
     has_js_framework = any(f in out["frameworks"] for f in ("react", "nextjs", "angular", "vue"))
     low_content = out["patterns"]["links_with_id"] < 5 and out["patterns"]["article_tags"] < 5 and len(html) < 200_000
-    out["is_likely_spa"] = has_js_framework and low_content
+    # Coquille vide : élément racine vide et scripts en modules, presque aucun mot visible. La taille du HTML ne
+    # compte pas : une feuille de style en ligne fait passer une coquille au-dessus de 200 Ko (audit du 04.11.2026).
+    out["visible_words"] = visible_words(html)
+    empty_root = bool(SPA_EMPTY_ROOT.search(html)) and 'type="module"' in html
+    shell = (has_js_framework or empty_root) and out["visible_words"] < SPA_MAX_WORDS
+    out["is_likely_spa"] = (has_js_framework and low_content and out["visible_words"] < SPA_MAX_WORDS * 4) or shell
 
     return out
 
@@ -1164,15 +1180,20 @@ def _decide_tier_doors(report: ProbeReport) -> tuple[int, str]:
         return 1, f"{len(t1['api_endpoints'])} endpoint(s) JSON trouvé(s)"
     if t1.get("rss_feeds"):
         return 1, f"{len(t1['rss_feeds'])} RSS/Atom feed(s) trouvé(s)"
+    if t2.get("is_likely_spa"):
+        # Le sitemap d'une SPA énumère des pages vides : la voie est l'API que l'application appelle.
+        return 4, (
+            "Site SPA : le HTML est une coquille vide ; chercher l'API de données dans le code de l'application "
+            "(scripts lus comme texte, jamais exécutés), sinon Chrome"
+            + (" ; le sitemap énumère seulement des adresses" if t1.get("sitemap") else "")
+        )
     if t1.get("sitemap"):
         return 1, "Sitemap XML disponible (utile pour énumération URLs)"
     if t2.get("inline_json"):
         return (
             2,
             f"Inline JSON détecté ({', '.join(j['marker'] for j in t2['inline_json'])})",
         )
-    if t2.get("is_likely_spa"):
-        return 4, "Site SPA sans alternative API — Chrome render nécessaire"
     if t2.get("fetched"):
         return 2, "HTML statique — extracteurs HTML/CSS classiques"
     return 4, "Aucune voie identifiée — tier 4 par défaut"

tests/test_spa_coquille.py

@@ -0,0 +1,48 @@
+"""A modern single-page application serves an empty shell: a mount element, module scripts, no text. Angular CLI
+builds no longer write ``ng-version`` in the served HTML, and inlined CSS can make the shell larger than 200 KB.
+The probe must still see a SPA, and a SPA must not be sent to its sitemap: the sitemap lists addresses whose pages
+are empty shells. The path is the data API in the application code."""
+
+import asyncio
+
+from accesspath import probe as probe_mod
+
+STYLE = "<style>" + ":root{--c:#fff}" * 15_000 + "</style>"  # inlined CSS, more than 200 KB
+
+
+def _shell(root: str) -> bytes:
+    return (
+        f'<!doctype html><html><head><meta charset="utf-8"><title>RPEX</title><base href="/">{STYLE}</head>'
+        f'<body>{root}<link rel="modulepreload" href="chunk-EUZTCWZM.js">'
+        '<script src="polyfills-ERLYOUQC.js" type="module"></script>'
+        '<script src="main-SSOFTLRP.js" type="module"></script></body></html>'
+    ).encode()
+
+
+def _tier2(body: bytes) -> dict:
+    return asyncio.run(probe_mod.discover_tier2("https://x.eu/doc", home={"ok": True, "body": body}))
+
+
+def test_an_angular_shell_without_ng_version_is_a_spa():
+    t2 = _tier2(_shell("<app-root></app-root>"))
+    assert "angular" in t2["frameworks"]
+    assert t2["is_likely_spa"] is True
+    assert t2["visible_words"] < 5
+
+
+def test_an_empty_mount_element_with_module_scripts_is_a_spa():
+    assert _tier2(_shell('<div id="app"></div>'))["is_likely_spa"] is True
+
+
+def test_a_page_with_text_is_not_a_spa():
+    texte = "<main>" + "<p>" + "Le texte de la page est servi par le serveur. " * 40 + "</p></main>"
+    assert _tier2(_shell("<app-root></app-root>" + texte))["is_likely_spa"] is False
+
+
+def test_a_spa_goes_to_its_api_and_not_to_its_sitemap():
+    r = probe_mod.ProbeReport(url="https://x.eu/doc", timestamp="t")
+    r.tier1 = {"sitemap": {"endpoint": "/sitemap.xml", "items": 8}}
+    r.tier2 = {"fetched": True, "is_likely_spa": True, "frameworks": ["angular"], "inline_json": []}
+    tier, why = probe_mod._decide_tier(r)
+    assert tier == 4
+    assert "API" in why and "code" in why and "sitemap" in why