From dbcecbde841b383ac2119b49e25dd0c2263dfec1 Mon Sep 17 00:00:00 2001 From: Binay Date: Tue, 18 Aug 2026 16:14:31 -0400 Subject: [PATCH] feat(corpus): XXE, path-traversal and Go SSRF cases + pinned real repos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #11, #12, #29, #13. Corpus goes 52 -> 60 cases: * LANG-53/54 XXE (CWE-611) in Java and PHP (#11) * LANG-56/57 path traversal (CWE-22) in Go and Java (#12) * LANG-59 SSRF (CWE-918) in Go (#29) * LANG-55/58/60 three SAFE decoys The decoys are the point as much as the positives: default-safe PHP XML parsing (entities are off by default on PHP 8+, so "parses XML" is not the bug), a constant filesystem path, and a constant host with a user-supplied query string. LANG-60 paid for itself immediately — it caught a false positive in signetry-core's brand-new go.taint.ssrf rule (constant host + tainted query string reported as SSRF), fixed in Signetry/core#97 before this landed. That is exactly the false-positive class the Python SSRF rule is held to, so the Go rule had to meet it too. Real repos (#13): OWASP WebGoat (Java) and OWASP RailsGoat (Ruby), both pinned to a full SHA. They are the first JVM/Ruby targets here — every existing case is Python or JavaScript, which under-exercises the multi-language tier. Fixes the pin mechanism while adding them. scan_real_repo cloned with --depth 1 then ran `git checkout ` with check=False. On a shallow clone the object is absent, so the checkout failed with fatal: unable to read tree the failure was swallowed, and the scan silently ran against the default-branch tip — so a case documented as "pinned for reproducibility" was not pinned. Verified both behaviours against a real shallow clone. Now fetches the object first and, if pinning truly cannot be honoured, records that in the note instead of passing an unpinned scan off as pinned. Requires signetry-core v0.7.0 (pin bumped): the new cases depend on its Go SSRF / Go+Java path-traversal / PHP XXE rules. Verified against the published tag in a clean environment — 60 cases, recall 1.0, 0 false positives; 50 tests and ruff green. --- CHANGELOG.md | 32 ++++ pyproject.toml | 2 +- signetry_eval/detection/corpus/multilang.py | 176 ++++++++++++++++++ .../detection/real_repo_benchmark.py | 34 +++- signetry_eval/detection/real_repos.py | 25 +++ tests/test_corpus_benchmark.py | 55 ++++++ uv.lock | 8 +- 7 files changed, 324 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9230aa..3c44d25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,38 @@ Until `1.0.0` the public API may change between minor versions. ## [Unreleased] +### Added — OWASP breadth in the detection corpus (52 → 60 cases) + +- **XXE (CWE-611)** in Java and PHP — `LANG-53`, `LANG-54` (eval#11). +- **Path traversal (CWE-22)** in Go and Java — `LANG-56`, `LANG-57` (eval#12). +- **SSRF (CWE-918)** in Go — `LANG-59` (eval#29), giving the class a second + language alongside Python. +- Three SAFE decoys probing the precision distinctions these rules must make: + default-safe PHP XML parsing (entities are off by default on PHP 8+), a constant + filesystem path, and a **constant host with a user-supplied query string** + (`LANG-55`, `LANG-58`, `LANG-60`). +- Two **pinned** real-repo cases — OWASP WebGoat (Java) and OWASP RailsGoat + (Ruby), the first JVM/Ruby targets here; every prior case is Python or + JavaScript (eval#13). + +The `LANG-60` decoy earned its keep immediately: it caught a false positive in +signetry-core's new Go SSRF rule, fixed in Signetry/core#97 before this landed. + +### Fixed — pinned real-repo cases were not actually pinned + +- `scan_real_repo` cloned with `--depth 1` and then ran `git checkout ` with + `check=False`. On a shallow clone the object is absent, so the checkout failed + (`fatal: unable to read tree`), the failure was swallowed, and the scan silently + ran against the **default-branch tip** — a case documented as "pinned for + reproducibility" was not pinned. Now fetches the specific object first, and if + pinning genuinely cannot be honoured it says so in the result note rather than + reporting an unpinned scan as pinned. + +### Changed + +- Pin `signetry-core` at `v0.7.0`; the corpus additions above depend on its new + Go SSRF / Go+Java path-traversal / PHP XXE rules. + ### Changed — Signetry rename (breaking) - Distribution `signetry-eval` and import package `signetry_eval`. The console diff --git a/pyproject.toml b/pyproject.toml index d1ae142..f4f76b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ "Topic :: Software Development :: Quality Assurance", ] dependencies = [ - "signetry-core @ git+https://github.com/Signetry/core@v0.6.0", + "signetry-core @ git+https://github.com/Signetry/core@v0.7.0", ] [project.urls] diff --git a/signetry_eval/detection/corpus/multilang.py b/signetry_eval/detection/corpus/multilang.py index b84c1fe..67918eb 100644 --- a/signetry_eval/detection/corpus/multilang.py +++ b/signetry_eval/detection/corpus/multilang.py @@ -251,6 +251,182 @@ def backup(params) ps.executeQuery(); } } +'''}, + expected=[], # SAFE + ), + + # --- OWASP breadth: XXE, path traversal, SSRF (Signetry/eval#11, #12, #29) --- + Case( + id="LANG-53-java-xxe", + family=Family.MULTILANG, + language="java", + title="Java: XML parsed with a default DocumentBuilderFactory (XXE)", + provenance="OWASP A05:2021 Security Misconfiguration; CWE-611. Pattern per the " + "OWASP XXE Prevention cheat sheet: the JAXP default factory resolves " + "external entities unless DOCTYPE processing is explicitly disabled.", + files={"XmlLoader.java": '''\ +import javax.xml.parsers.DocumentBuilderFactory; +import org.xml.sax.InputSource; +import java.io.StringReader; + +public class XmlLoader { + public void load(String xml) throws Exception { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.newDocumentBuilder().parse(new InputSource(new StringReader(xml))); + } +} +'''}, + expected=[ExpectedFinding("CWE-611", "xxe", "XmlLoader.java")], + ), + Case( + id="LANG-54-php-xxe", + family=Family.MULTILANG, + language="php", + title="PHP: loadXML with LIBXML_NOENT|LIBXML_DTDLOAD (XXE)", + provenance="OWASP A05:2021; CWE-611. Since PHP 8 / libxml 2.9 external entities " + "are off by default, so the vulnerable pattern is code that explicitly " + "re-enables them via LIBXML_NOENT / LIBXML_DTDLOAD.", + files={"import.php": '''\ +loadXML($xml, LIBXML_NOENT | LIBXML_DTDLOAD); +echo $doc->saveXML(); +'''}, + expected=[ExpectedFinding("CWE-611", "xxe", "import.php")], + ), + Case( + id="LANG-55-SAFE-php-xml-default", + family=Family.MULTILANG, + language="php", + title="SAFE: PHP loadXML without entity flags (default-safe on modern PHP)", + provenance="Crafted SAFE decoy: parsing untrusted XML is not itself XXE on PHP 8+ " + "(entities off by default) — probes whether the rule keys on the flag " + "rather than on 'parses XML'.", + files={"safe_import.php": '''\ +loadXML($xml); +echo $doc->saveXML(); +'''}, + expected=[], # SAFE + ), + Case( + id="LANG-56-go-path-traversal", + family=Family.MULTILANG, + language="go", + title="Go: file path built from a query parameter (traversal)", + provenance="OWASP A01:2021 Broken Access Control; CWE-22. Unconfined path join " + "from a request parameter, per the OWASP Path Traversal description.", + files={"files.go": '''\ +package main + +import ( + "net/http" + "os" +) + +func download(w http.ResponseWriter, r *http.Request) { + name := r.URL.Query().Get("file") + data, err := os.ReadFile("/var/data/" + name) + if err != nil { + http.Error(w, "not found", 404) + return + } + w.Write(data) +} +'''}, + expected=[ExpectedFinding("CWE-22", "path_traversal", "files.go")], + ), + Case( + id="LANG-57-java-path-traversal", + family=Family.MULTILANG, + language="java", + title="Java: FileInputStream opened on a request parameter (traversal)", + provenance="OWASP A01:2021; CWE-22. Servlet parameter concatenated into a " + "filesystem path with no canonicalisation or base-dir confinement.", + files={"Download.java": '''\ +import javax.servlet.http.HttpServletRequest; + +public class Download { + public void send(HttpServletRequest req) throws Exception { + String name = req.getParameter("file"); + java.io.FileInputStream in = new java.io.FileInputStream("/var/data/" + name); + in.close(); + } +} +'''}, + expected=[ExpectedFinding("CWE-22", "path_traversal", "Download.java")], + ), + Case( + id="LANG-58-SAFE-go-constant-path", + family=Family.MULTILANG, + language="go", + title="SAFE: Go reads a compiled-in constant path", + provenance="Crafted SAFE decoy: a constant filesystem path is not traversal " + "(false-positive probe for the traversal sink).", + files={"config.go": '''\ +package main + +import "os" + +func loadConfig() ([]byte, error) { + return os.ReadFile("/etc/app/config.yaml") +} +'''}, + expected=[], # SAFE + ), + Case( + id="LANG-59-go-ssrf", + family=Family.MULTILANG, + language="go", + title="Go: outbound HTTP request to a user-controlled URL (SSRF)", + provenance="OWASP A10:2021 Server-Side Request Forgery; CWE-918. A fetch-by-URL " + "handler forwards a request parameter straight to http.Get, reaching " + "internal services and cloud metadata endpoints.", + files={"proxy.go": '''\ +package main + +import ( + "io" + "net/http" +) + +func fetch(w http.ResponseWriter, r *http.Request) { + target := r.URL.Query().Get("url") + resp, err := http.Get(target) + if err != nil { + http.Error(w, "fetch failed", 502) + return + } + defer resp.Body.Close() + io.Copy(w, resp.Body) +} +'''}, + expected=[ExpectedFinding("CWE-918", "ssrf", "proxy.go")], + ), + Case( + id="LANG-60-SAFE-go-constant-url", + family=Family.MULTILANG, + language="go", + title="SAFE: Go fetches a constant URL with a user-supplied query string", + provenance="Crafted SAFE decoy: the destination host is compiled in and only the " + "query string is user-controlled — not SSRF. Probes the same " + "constant-URL-with-tainted-parameters distinction that the Python SSRF " + "rule is measured on.", + files={"client.go": '''\ +package main + +import ( + "net/http" + "net/url" +) + +func search(r *http.Request) (*http.Response, error) { + q := r.URL.Query().Get("q") + return http.Get("https://api.example.com/search?q=" + url.QueryEscape(q)) +} '''}, expected=[], # SAFE ), diff --git a/signetry_eval/detection/real_repo_benchmark.py b/signetry_eval/detection/real_repo_benchmark.py index c389b37..33d5194 100644 --- a/signetry_eval/detection/real_repo_benchmark.py +++ b/signetry_eval/detection/real_repo_benchmark.py @@ -40,6 +40,31 @@ def _git_available() -> bool: return False +def _checkout_pinned(root: Path, commit: str) -> str: + """Check out a pinned commit in a (possibly shallow) clone. + + ``resolve_scan_target`` clones with ``--depth 1``, so the pinned object is not + present and a plain ``git checkout `` fails. Previously that failure was + swallowed (``check=False``) and the scan silently ran against the default-branch + tip — so a "pinned for reproducibility" case was not actually pinned. + + Fetch the specific object first (GitHub permits fetching an arbitrary SHA), and + if pinning genuinely cannot be honoured, say so in the note rather than + reporting an unpinned scan as pinned. + """ + fetched = subprocess.run( + ["git", "fetch", "--quiet", "--depth", "1", "origin", commit], + cwd=root, capture_output=True, check=False, + ) + ref = "FETCH_HEAD" if fetched.returncode == 0 else commit + checked = subprocess.run( + ["git", "checkout", "-q", ref], cwd=root, capture_output=True, check=False, + ) + if checked.returncode != 0: + return f"NOT pinned (commit {commit[:12]} unreachable; scanned default branch)" + return f"pinned @ {commit[:12]}" + + def scan_real_repo(case: RealRepoCase, *, use_semgrep: bool = False, depth: int = 1) -> RealRepoResult: """Clone + scan one real repo. Returns a result; never raises (records notes).""" from signetry_core import scan_repository @@ -47,22 +72,25 @@ def scan_real_repo(case: RealRepoCase, *, use_semgrep: bool = False, depth: int if not _git_available(): return RealRepoResult(case.id, case.url, ran=False, note="git unavailable") + pin_note = "" try: with resolve_scan_target(case.url, depth=depth) as root: root = Path(root) if case.commit: - subprocess.run(["git", "checkout", "-q", case.commit], cwd=root, - capture_output=True, check=False) + pin_note = _checkout_pinned(root, case.commit) report = scan_repository(root, use_semgrep=use_semgrep) except RuntimeError as exc: return RealRepoResult(case.id, case.url, ran=False, note=f"clone/scan failed: {exc}") by_cat: dict[str, int] = {} for f in report.findings: by_cat[f.category] = by_cat.get(f.category, 0) + 1 + note = f"layers: {', '.join(report.layers)}" + if pin_note: + note = f"{pin_note} · {note}" return RealRepoResult( case.id, case.url, ran=True, files_scanned=report.files_scanned, total_findings=len(report.findings), by_category=by_cat, - note=f"layers: {', '.join(report.layers)}", + note=note, ) diff --git a/signetry_eval/detection/real_repos.py b/signetry_eval/detection/real_repos.py index 900d3cc..b65c4c0 100644 --- a/signetry_eval/detection/real_repos.py +++ b/signetry_eval/detection/real_repos.py @@ -77,4 +77,29 @@ class RealRepoCase: "Large; scanned shallowly. Expected: assorted JS/TS sinks.", expect_at_least=[], ), + # The cases above pin no commit, so they drift with their default branch. The two + # below are pinned, and are the first JVM/Ruby targets here — every case above is + # Python or JavaScript, which under-exercises the multi-language tier. + RealRepoCase( + id="REAL-webgoat", + url="https://github.com/WebGoat/WebGoat.git", + commit="7517acca95d9851da706452454c223dd13545ef4", + languages=["java"], + provenance="OWASP WebGoat — the reference deliberately-insecure Java teaching " + "app, maintained by OWASP. Pinned for reproducibility. Exercises the " + "Java tier (JDBC concatenation, XXE, native deserialization, path " + "traversal) that no other real-repo case here reaches.", + expect_at_least=[], # reported, not asserted (see the module docstring) + ), + RealRepoCase( + id="REAL-railsgoat", + url="https://github.com/OWASP/railsgoat.git", + commit="0222f7da3406ba3ab637bc6d24ae9366b5f0a680", + languages=["ruby"], + provenance="OWASP RailsGoat — deliberately vulnerable Rails app covering the " + "OWASP Top 10. Pinned for reproducibility. First Ruby target here, so " + "it exercises the Ruby rules (interpolated SQL/command, YAML/Marshal " + "load) against real application code rather than snippets.", + expect_at_least=[], + ), ] diff --git a/tests/test_corpus_benchmark.py b/tests/test_corpus_benchmark.py index e68aded..6ed1ddd 100644 --- a/tests/test_corpus_benchmark.py +++ b/tests/test_corpus_benchmark.py @@ -193,3 +193,58 @@ def test_corpus_markdown_renders(): assert "public test cases" in md assert "Recall by language" in md assert "signetry-core" in md + + +# --- OWASP breadth additions (eval#11 XXE, #12 path traversal, #29 Go SSRF) --- + + +def test_corpus_includes_xxe_traversal_ssrf_cases(): + ids = {c.id for c in ALL_CASES} + assert {"LANG-53-java-xxe", "LANG-54-php-xxe", "LANG-56-go-path-traversal", + "LANG-57-java-path-traversal", "LANG-59-go-ssrf"} <= ids + + +def test_corpus_covers_xxe_traversal_ssrf_in_multiple_languages(): + """Each of the three classes must be represented in at least two languages, so + the corpus measures breadth rather than one language's rule.""" + by_class: dict[str, set[str]] = {} + for c in ALL_CASES: + for e in c.expected: + by_class.setdefault(e.category, set()).add(c.language) + assert {"java", "php"} <= by_class.get("xxe", set()) + assert {"go", "java"} <= by_class.get("path_traversal", set()) + assert {"go", "python"} <= by_class.get("ssrf", set()) + + +@requires_engine +def test_signetry_detects_new_owasp_breadth_cases(): + score = run_corpus_benchmark("signetry-core", signetry_corpus_adapter()) + for cid in ("LANG-53-java-xxe", "LANG-54-php-xxe", "LANG-56-go-path-traversal", + "LANG-57-java-path-traversal", "LANG-59-go-ssrf"): + c = next(x for x in score.cases if x.case_id == cid) + assert c.detected == c.expected >= 1, f"{cid} not detected" + + +@requires_engine +def test_signetry_no_fp_on_new_safe_decoys(): + """The safe decoys probe the precision distinctions these rules must make: + default-safe PHP XML parsing, a constant filesystem path, and a constant host + with a user-supplied query string.""" + score = run_corpus_benchmark("signetry-core", signetry_corpus_adapter()) + for cid in ("LANG-55-SAFE-php-xml-default", "LANG-58-SAFE-go-constant-path", + "LANG-60-SAFE-go-constant-url"): + c = next(x for x in score.cases if x.case_id == cid) + assert c.false_positives == 0, f"false positive on {cid}" + + +def test_pinned_real_repo_cases_exist_and_are_pinned(): + from signetry_eval.detection.real_repos import REAL_REPO_CASES + + pinned = {c.id: c for c in REAL_REPO_CASES if c.commit} + assert {"REAL-webgoat", "REAL-railsgoat"} <= set(pinned) + for c in pinned.values(): + assert len(c.commit) == 40, f"{c.id}: pin must be a full 40-char SHA" + assert c.provenance and len(c.provenance) > 20 + # These are the first JVM/Ruby real-repo targets. + langs = {lang for c in REAL_REPO_CASES for lang in c.languages} + assert {"java", "ruby"} <= langs diff --git a/uv.lock b/uv.lock index 5a88297..58c3c44 100644 --- a/uv.lock +++ b/uv.lock @@ -308,8 +308,8 @@ wheels = [ [[package]] name = "signetry-core" -version = "0.6.0" -source = { git = "https://github.com/Signetry/core?rev=v0.6.0#b1f437ec33ca3e57586abc56accda7de529ec519" } +version = "0.7.0" +source = { git = "https://github.com/Signetry/core?rev=v0.7.0#0d39eb34f32152f5a3015ce9282245e38c25d9fc" } dependencies = [ { name = "cryptography" }, { name = "pyyaml" }, @@ -317,7 +317,7 @@ dependencies = [ [[package]] name = "signetry-eval" -version = "0.2.2" +version = "0.2.3" source = { editable = "." } dependencies = [ { name = "signetry-core" }, @@ -333,6 +333,6 @@ dev = [ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.6.0" }, - { name = "signetry-core", git = "https://github.com/Signetry/core?rev=v0.6.0" }, + { name = "signetry-core", git = "https://github.com/Signetry/core?rev=v0.7.0" }, ] provides-extras = ["dev"]