feat(corpus): XXE, path-traversal and Go SSRF cases + pinned real repos

CHANGELOG.md

@@ -5,6 +5,38 @@ Until `1.0.0` the public API may change between minor versions.
 
 ## [Unreleased]
 
+### Added — OWASP breadth in the detection corpus (52 → 60 cases)
+
+- **XXE (CWE-611)** in Java and PHP — `LANG-53`, `LANG-54` (eval#11).
+- **Path traversal (CWE-22)** in Go and Java — `LANG-56`, `LANG-57` (eval#12).
+- **SSRF (CWE-918)** in Go — `LANG-59` (eval#29), giving the class a second
+  language alongside Python.
+- Three SAFE decoys probing the precision distinctions these rules must make:
+  default-safe PHP XML parsing (entities are off by default on PHP 8+), a constant
+  filesystem path, and a **constant host with a user-supplied query string**
+  (`LANG-55`, `LANG-58`, `LANG-60`).
+- Two **pinned** real-repo cases — OWASP WebGoat (Java) and OWASP RailsGoat
+  (Ruby), the first JVM/Ruby targets here; every prior case is Python or
+  JavaScript (eval#13).
+
+The `LANG-60` decoy earned its keep immediately: it caught a false positive in
+signetry-core's new Go SSRF rule, fixed in Signetry/core#97 before this landed.
+
+### Fixed — pinned real-repo cases were not actually pinned
+
+- `scan_real_repo` cloned with `--depth 1` and then ran `git checkout <sha>` with
+  `check=False`. On a shallow clone the object is absent, so the checkout failed
+  (`fatal: unable to read tree`), the failure was swallowed, and the scan silently
+  ran against the **default-branch tip** — a case documented as "pinned for
+  reproducibility" was not pinned. Now fetches the specific object first, and if
+  pinning genuinely cannot be honoured it says so in the result note rather than
+  reporting an unpinned scan as pinned.
+
+### Changed
+
+- Pin `signetry-core` at `v0.7.0`; the corpus additions above depend on its new
+  Go SSRF / Go+Java path-traversal / PHP XXE rules.
+
 ### Changed — Signetry rename (breaking)
 
 - Distribution `signetry-eval` and import package `signetry_eval`. The console

pyproject.toml

@@ -21,7 +21,7 @@ classifiers = [
     "Topic :: Software Development :: Quality Assurance",
 ]
 dependencies = [
-    "signetry-core @ git+https://github.com/Signetry/core@v0.6.0",
+    "signetry-core @ git+https://github.com/Signetry/core@v0.7.0",
 ]
 
 [project.urls]

signetry_eval/detection/corpus/multilang.py

@@ -251,6 +251,182 @@ def backup(params)
         ps.executeQuery();
     }
 }
+'''},
+        expected=[],  # SAFE
+    ),
+
+    # --- OWASP breadth: XXE, path traversal, SSRF (Signetry/eval#11, #12, #29) ---
+    Case(
+        id="LANG-53-java-xxe",
+        family=Family.MULTILANG,
+        language="java",
+        title="Java: XML parsed with a default DocumentBuilderFactory (XXE)",
+        provenance="OWASP A05:2021 Security Misconfiguration; CWE-611. Pattern per the "
+                   "OWASP XXE Prevention cheat sheet: the JAXP default factory resolves "
+                   "external entities unless DOCTYPE processing is explicitly disabled.",
+        files={"XmlLoader.java": '''\
+import javax.xml.parsers.DocumentBuilderFactory;
+import org.xml.sax.InputSource;
+import java.io.StringReader;
+
+public class XmlLoader {
+    public void load(String xml) throws Exception {
+        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+        factory.newDocumentBuilder().parse(new InputSource(new StringReader(xml)));
+    }
+}
+'''},
+        expected=[ExpectedFinding("CWE-611", "xxe", "XmlLoader.java")],
+    ),
+    Case(
+        id="LANG-54-php-xxe",
+        family=Family.MULTILANG,
+        language="php",
+        title="PHP: loadXML with LIBXML_NOENT|LIBXML_DTDLOAD (XXE)",
+        provenance="OWASP A05:2021; CWE-611. Since PHP 8 / libxml 2.9 external entities "
+                   "are off by default, so the vulnerable pattern is code that explicitly "
+                   "re-enables them via LIBXML_NOENT / LIBXML_DTDLOAD.",
+        files={"import.php": '''\
+<?php
+$xml = $_POST["payload"];
+$doc = new DOMDocument();
+$doc->loadXML($xml, LIBXML_NOENT | LIBXML_DTDLOAD);
+echo $doc->saveXML();
+'''},
+        expected=[ExpectedFinding("CWE-611", "xxe", "import.php")],
+    ),
+    Case(
+        id="LANG-55-SAFE-php-xml-default",
+        family=Family.MULTILANG,
+        language="php",
+        title="SAFE: PHP loadXML without entity flags (default-safe on modern PHP)",
+        provenance="Crafted SAFE decoy: parsing untrusted XML is not itself XXE on PHP 8+ "
+                   "(entities off by default) — probes whether the rule keys on the flag "
+                   "rather than on 'parses XML'.",
+        files={"safe_import.php": '''\
+<?php
+$xml = $_POST["payload"];
+$doc = new DOMDocument();
+$doc->loadXML($xml);
+echo $doc->saveXML();
+'''},
+        expected=[],  # SAFE
+    ),
+    Case(
+        id="LANG-56-go-path-traversal",
+        family=Family.MULTILANG,
+        language="go",
+        title="Go: file path built from a query parameter (traversal)",
+        provenance="OWASP A01:2021 Broken Access Control; CWE-22. Unconfined path join "
+                   "from a request parameter, per the OWASP Path Traversal description.",
+        files={"files.go": '''\
+package main
+
+import (
+    "net/http"
+    "os"
+)
+
+func download(w http.ResponseWriter, r *http.Request) {
+    name := r.URL.Query().Get("file")
+    data, err := os.ReadFile("/var/data/" + name)
+    if err != nil {
+        http.Error(w, "not found", 404)
+        return
+    }
+    w.Write(data)
+}
+'''},
+        expected=[ExpectedFinding("CWE-22", "path_traversal", "files.go")],
+    ),
+    Case(
+        id="LANG-57-java-path-traversal",
+        family=Family.MULTILANG,
+        language="java",
+        title="Java: FileInputStream opened on a request parameter (traversal)",
+        provenance="OWASP A01:2021; CWE-22. Servlet parameter concatenated into a "
+                   "filesystem path with no canonicalisation or base-dir confinement.",
+        files={"Download.java": '''\
+import javax.servlet.http.HttpServletRequest;
+
+public class Download {
+    public void send(HttpServletRequest req) throws Exception {
+        String name = req.getParameter("file");
+        java.io.FileInputStream in = new java.io.FileInputStream("/var/data/" + name);
+        in.close();
+    }
+}
+'''},
+        expected=[ExpectedFinding("CWE-22", "path_traversal", "Download.java")],
+    ),
+    Case(
+        id="LANG-58-SAFE-go-constant-path",
+        family=Family.MULTILANG,
+        language="go",
+        title="SAFE: Go reads a compiled-in constant path",
+        provenance="Crafted SAFE decoy: a constant filesystem path is not traversal "
+                   "(false-positive probe for the traversal sink).",
+        files={"config.go": '''\
+package main
+
+import "os"
+
+func loadConfig() ([]byte, error) {
+    return os.ReadFile("/etc/app/config.yaml")
+}
+'''},
+        expected=[],  # SAFE
+    ),
+    Case(
+        id="LANG-59-go-ssrf",
+        family=Family.MULTILANG,
+        language="go",
+        title="Go: outbound HTTP request to a user-controlled URL (SSRF)",
+        provenance="OWASP A10:2021 Server-Side Request Forgery; CWE-918. A fetch-by-URL "
+                   "handler forwards a request parameter straight to http.Get, reaching "
+                   "internal services and cloud metadata endpoints.",
+        files={"proxy.go": '''\
+package main
+
+import (
+    "io"
+    "net/http"
+)
+
+func fetch(w http.ResponseWriter, r *http.Request) {
+    target := r.URL.Query().Get("url")
+    resp, err := http.Get(target)
+    if err != nil {
+        http.Error(w, "fetch failed", 502)
+        return
+    }
+    defer resp.Body.Close()
+    io.Copy(w, resp.Body)
+}
+'''},
+        expected=[ExpectedFinding("CWE-918", "ssrf", "proxy.go")],
+    ),
+    Case(
+        id="LANG-60-SAFE-go-constant-url",
+        family=Family.MULTILANG,
+        language="go",
+        title="SAFE: Go fetches a constant URL with a user-supplied query string",
+        provenance="Crafted SAFE decoy: the destination host is compiled in and only the "
+                   "query string is user-controlled — not SSRF. Probes the same "
+                   "constant-URL-with-tainted-parameters distinction that the Python SSRF "
+                   "rule is measured on.",
+        files={"client.go": '''\
+package main
+
+import (
+    "net/http"
+    "net/url"
+)
+
+func search(r *http.Request) (*http.Response, error) {
+    q := r.URL.Query().Get("q")
+    return http.Get("https://api.example.com/search?q=" + url.QueryEscape(q))
+}
 '''},
         expected=[],  # SAFE
     ),

signetry_eval/detection/real_repo_benchmark.py

@@ -40,29 +40,57 @@ def _git_available() -> bool:
         return False
 
 
+def _checkout_pinned(root: Path, commit: str) -> str:
+    """Check out a pinned commit in a (possibly shallow) clone.
+
+    ``resolve_scan_target`` clones with ``--depth 1``, so the pinned object is not
+    present and a plain ``git checkout <sha>`` fails. Previously that failure was
+    swallowed (``check=False``) and the scan silently ran against the default-branch
+    tip — so a "pinned for reproducibility" case was not actually pinned.
+
+    Fetch the specific object first (GitHub permits fetching an arbitrary SHA), and
+    if pinning genuinely cannot be honoured, say so in the note rather than
+    reporting an unpinned scan as pinned.
+    """
+    fetched = subprocess.run(
+        ["git", "fetch", "--quiet", "--depth", "1", "origin", commit],
+        cwd=root, capture_output=True, check=False,
+    )
+    ref = "FETCH_HEAD" if fetched.returncode == 0 else commit
+    checked = subprocess.run(
+        ["git", "checkout", "-q", ref], cwd=root, capture_output=True, check=False,
+    )
+    if checked.returncode != 0:
+        return f"NOT pinned (commit {commit[:12]} unreachable; scanned default branch)"
+    return f"pinned @ {commit[:12]}"
+
+
 def scan_real_repo(case: RealRepoCase, *, use_semgrep: bool = False, depth: int = 1) -> RealRepoResult:
     """Clone + scan one real repo. Returns a result; never raises (records notes)."""
     from signetry_core import scan_repository
     from signetry_core.pipeline.findings.fetch import resolve_scan_target
 
     if not _git_available():
         return RealRepoResult(case.id, case.url, ran=False, note="git unavailable")
+    pin_note = ""
     try:
         with resolve_scan_target(case.url, depth=depth) as root:
             root = Path(root)
             if case.commit:
-                subprocess.run(["git", "checkout", "-q", case.commit], cwd=root,
-                               capture_output=True, check=False)
+                pin_note = _checkout_pinned(root, case.commit)
             report = scan_repository(root, use_semgrep=use_semgrep)
     except RuntimeError as exc:
         return RealRepoResult(case.id, case.url, ran=False, note=f"clone/scan failed: {exc}")
     by_cat: dict[str, int] = {}
     for f in report.findings:
         by_cat[f.category] = by_cat.get(f.category, 0) + 1
+    note = f"layers: {', '.join(report.layers)}"
+    if pin_note:
+        note = f"{pin_note} · {note}"
     return RealRepoResult(
         case.id, case.url, ran=True, files_scanned=report.files_scanned,
         total_findings=len(report.findings), by_category=by_cat,
-        note=f"layers: {', '.join(report.layers)}",
+        note=note,
     )
 
 

signetry_eval/detection/real_repos.py

@@ -77,4 +77,29 @@ class RealRepoCase:
                    "Large; scanned shallowly. Expected: assorted JS/TS sinks.",
         expect_at_least=[],
     ),
+    # The cases above pin no commit, so they drift with their default branch. The two
+    # below are pinned, and are the first JVM/Ruby targets here — every case above is
+    # Python or JavaScript, which under-exercises the multi-language tier.
+    RealRepoCase(
+        id="REAL-webgoat",
+        url="https://github.com/WebGoat/WebGoat.git",
+        commit="7517acca95d9851da706452454c223dd13545ef4",
+        languages=["java"],
+        provenance="OWASP WebGoat — the reference deliberately-insecure Java teaching "
+                   "app, maintained by OWASP. Pinned for reproducibility. Exercises the "
+                   "Java tier (JDBC concatenation, XXE, native deserialization, path "
+                   "traversal) that no other real-repo case here reaches.",
+        expect_at_least=[],  # reported, not asserted (see the module docstring)
+    ),
+    RealRepoCase(
+        id="REAL-railsgoat",
+        url="https://github.com/OWASP/railsgoat.git",
+        commit="0222f7da3406ba3ab637bc6d24ae9366b5f0a680",
+        languages=["ruby"],
+        provenance="OWASP RailsGoat — deliberately vulnerable Rails app covering the "
+                   "OWASP Top 10. Pinned for reproducibility. First Ruby target here, so "
+                   "it exercises the Ruby rules (interpolated SQL/command, YAML/Marshal "
+                   "load) against real application code rather than snippets.",
+        expect_at_least=[],
+    ),
 ]

tests/test_corpus_benchmark.py

@@ -193,3 +193,58 @@ def test_corpus_markdown_renders():
     assert "public test cases" in md
     assert "Recall by language" in md
     assert "signetry-core" in md
+
+
+# --- OWASP breadth additions (eval#11 XXE, #12 path traversal, #29 Go SSRF) ---
+
+
+def test_corpus_includes_xxe_traversal_ssrf_cases():
+    ids = {c.id for c in ALL_CASES}
+    assert {"LANG-53-java-xxe", "LANG-54-php-xxe", "LANG-56-go-path-traversal",
+            "LANG-57-java-path-traversal", "LANG-59-go-ssrf"} <= ids
+
+
+def test_corpus_covers_xxe_traversal_ssrf_in_multiple_languages():
+    """Each of the three classes must be represented in at least two languages, so
+    the corpus measures breadth rather than one language's rule."""
+    by_class: dict[str, set[str]] = {}
+    for c in ALL_CASES:
+        for e in c.expected:
+            by_class.setdefault(e.category, set()).add(c.language)
+    assert {"java", "php"} <= by_class.get("xxe", set())
+    assert {"go", "java"} <= by_class.get("path_traversal", set())
+    assert {"go", "python"} <= by_class.get("ssrf", set())
+
+
+@requires_engine
+def test_signetry_detects_new_owasp_breadth_cases():
+    score = run_corpus_benchmark("signetry-core", signetry_corpus_adapter())
+    for cid in ("LANG-53-java-xxe", "LANG-54-php-xxe", "LANG-56-go-path-traversal",
+                "LANG-57-java-path-traversal", "LANG-59-go-ssrf"):
+        c = next(x for x in score.cases if x.case_id == cid)
+        assert c.detected == c.expected >= 1, f"{cid} not detected"
+
+
+@requires_engine
+def test_signetry_no_fp_on_new_safe_decoys():
+    """The safe decoys probe the precision distinctions these rules must make:
+    default-safe PHP XML parsing, a constant filesystem path, and a constant host
+    with a user-supplied query string."""
+    score = run_corpus_benchmark("signetry-core", signetry_corpus_adapter())
+    for cid in ("LANG-55-SAFE-php-xml-default", "LANG-58-SAFE-go-constant-path",
+                "LANG-60-SAFE-go-constant-url"):
+        c = next(x for x in score.cases if x.case_id == cid)
+        assert c.false_positives == 0, f"false positive on {cid}"
+
+
+def test_pinned_real_repo_cases_exist_and_are_pinned():
+    from signetry_eval.detection.real_repos import REAL_REPO_CASES
+
+    pinned = {c.id: c for c in REAL_REPO_CASES if c.commit}
+    assert {"REAL-webgoat", "REAL-railsgoat"} <= set(pinned)
+    for c in pinned.values():
+        assert len(c.commit) == 40, f"{c.id}: pin must be a full 40-char SHA"
+        assert c.provenance and len(c.provenance) > 20
+    # These are the first JVM/Ruby real-repo targets.
+    langs = {lang for c in REAL_REPO_CASES for lang in c.languages}
+    assert {"java", "ruby"} <= langs