From dde90694fb1a9662bb978c51a5078a5b6468bb93 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 08:26:50 +0000 Subject: [PATCH 1/6] feat(idris-dependents): read-only audit for external Idris2 dependents Port of a Bun script that searches GitHub code search for other people's Idris2 projects depending on our packages, rewritten as a standalone Rust crate (not a bots/ slot: it has no BotId, per invariant 6). The .ipkg reader now follows Idris2's own package lexer instead of a regex chain, which fixes dependencies the script silently dropped: - CRLF files lost every dependency - an `==` bound on a continuation line ended the field early - `{-` inside a `--` comment swallowed real lines - pack.toml urls with `#` or single quotes were missed (now a real TOML parser) Requests are GET-only by construction (the Io trait has only `get`), and queries GitHub truncates at 1000 results are reported, not dropped. Tests: the original suite ported, regressions for each bug, a 2000-case proptest round-trip (the old parser fails 404 of 500 generated files), and an opt-in differential test against `idris2 --dump-ipkg-json`. Added to the rust.yml build/test/clippy/fmt matrix and a Justfile recipe. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01J4QvDfb1gn2SD218jMg9jk --- .github/workflows/rust.yml | 2 + Justfile | 5 + idris-dependents/.gitignore | 2 + idris-dependents/Cargo.lock | 1689 +++++++++++++++++ idris-dependents/Cargo.toml | 26 + idris-dependents/README.adoc | 74 + idris-dependents/src/catalogue.rs | 73 + idris-dependents/src/ipkg.rs | 285 +++ idris-dependents/src/lib.rs | 12 + idris-dependents/src/main.rs | 203 ++ idris-dependents/src/refs.rs | 80 + idris-dependents/src/scan.rs | 301 +++ idris-dependents/tests/common/mod.rs | 156 ++ idris-dependents/tests/dump_samples.rs | 30 + idris-dependents/tests/fixtures/ochrance.ipkg | 32 + .../tests/fixtures/ordering-laws.ipkg | 15 + idris-dependents/tests/fixtures/pack.toml | 18 + idris-dependents/tests/fixtures/proven.ipkg | 378 ++++ idris-dependents/tests/oracle.rs | 101 + idris-dependents/tests/roundtrip_props.rs | 26 + idris-dependents/tests/rules_tests.rs | 306 +++ idris-dependents/tests/scan_tests.rs | 217 +++ 22 files changed, 4031 insertions(+) create mode 100644 idris-dependents/.gitignore create mode 100644 idris-dependents/Cargo.lock create mode 100644 idris-dependents/Cargo.toml create mode 100644 idris-dependents/README.adoc create mode 100644 idris-dependents/src/catalogue.rs create mode 100644 idris-dependents/src/ipkg.rs create mode 100644 idris-dependents/src/lib.rs create mode 100644 idris-dependents/src/main.rs create mode 100644 idris-dependents/src/refs.rs create mode 100644 idris-dependents/src/scan.rs create mode 100644 idris-dependents/tests/common/mod.rs create mode 100644 idris-dependents/tests/dump_samples.rs create mode 100644 idris-dependents/tests/fixtures/ochrance.ipkg create mode 100644 idris-dependents/tests/fixtures/ordering-laws.ipkg create mode 100644 idris-dependents/tests/fixtures/pack.toml create mode 100644 idris-dependents/tests/fixtures/proven.ipkg create mode 100644 idris-dependents/tests/oracle.rs create mode 100644 idris-dependents/tests/roundtrip_props.rs create mode 100644 idris-dependents/tests/rules_tests.rs create mode 100644 idris-dependents/tests/scan_tests.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b6cc0357..4996e22c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -45,6 +45,8 @@ jobs: dir: dashboard - module: rhodibot dir: bots/rhodibot + - module: idris-dependents + dir: idris-dependents defaults: run: working-directory: ${{ matrix.dir }} diff --git a/Justfile b/Justfile index 2c3b52ec..d65b4b1c 100644 --- a/Justfile +++ b/Justfile @@ -113,6 +113,11 @@ validate-rsr-drift: refresh-rsr-census: @bash scripts/rsr-census.sh --refresh +# Read-only audit: external Idris2 projects depending on our packages (GitHub +# code search, GET only; needs GITHUB_TOKEN). Evidence goes to OUT. +audit-idris-dependents OUT="idris-dependents.json" *ARGS: + cd idris-dependents && cargo run --release -- scan --json "{{invocation_directory()}}/{{OUT}}" {{ARGS}} + # Clean all build artifacts clean: cd robot-repo-automaton && cargo clean diff --git a/idris-dependents/.gitignore b/idris-dependents/.gitignore new file mode 100644 index 00000000..a3c49614 --- /dev/null +++ b/idris-dependents/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: MPL-2.0 +/target/ diff --git a/idris-dependents/Cargo.lock b/idris-dependents/Cargo.lock new file mode 100644 index 00000000..3e96b6e8 --- /dev/null +++ b/idris-dependents/Cargo.lock @@ -0,0 +1,1689 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bit-set" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269" +dependencies = [ + "borsh", + "serde", +] + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" + +[[package]] +name = "icu_properties" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" + +[[package]] +name = "icu_provider" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "idris-dependents" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "proptest", + "reqwest", + "serde", + "serde_json", + "toml", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags", + "chacha20", + "core_detect", + "num-traits", + "rand", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_xorshift" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf" +dependencies = [ + "rand_core", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7523beb55eece201a2356bee0bbca0d1ab466c14c07703b2e0ee6d42cb0c2c" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "toml" +version = "1.1.8+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20489e00e4d8741d6be680764cc12e270655e375a20d1011e844a9c3379e678d" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.17+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770" + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/idris-dependents/Cargo.toml b/idris-dependents/Cargo.toml new file mode 100644 index 00000000..9eb053d0 --- /dev/null +++ b/idris-dependents/Cargo.toml @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: MPL-2.0 +[package] +name = "idris-dependents" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" +authors = ["Jonathan D.A. Jewell "] +description = "Read-only estate audit: find external Idris2 projects that depend on hyperpolymath packages" +repository = "https://github.com/hyperpolymath/gitbot-fleet" +keywords = ["idris2", "audit", "dependents", "gitbot-fleet"] +categories = ["development-tools"] + +[[bin]] +name = "idris-dependents" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0.102" +clap = { version = "4.6.1", features = ["derive"] } +reqwest = { version = "0.12.28", features = ["blocking", "rustls-tls"], default-features = false } +serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0.150" +toml = { version = "1.1.2", features = ["preserve_order"] } + +[dev-dependencies] +proptest = "1.7.0" diff --git a/idris-dependents/README.adoc b/idris-dependents/README.adoc new file mode 100644 index 00000000..55f86306 --- /dev/null +++ b/idris-dependents/README.adoc @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: MPL-2.0 +// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell += idris-dependents +:icons: font + +A read-only estate audit: find people *outside* our own accounts whose Idris2 +projects depend on our packages, using GitHub code search. It sends GET requests +only. The `scan::Io` trait it talks through has no other verb, so it cannot +write to GitHub. + +It is a standalone audit crate, not a fleet bot: it has no `BotId` and does not +route through `shared-context` (see invariant 6 in `.claude/CLAUDE.md`). + +== Usage + +[source,bash] +---- +GITHUB_TOKEN=$(gh auth token) just audit-idris-dependents +# or +cd idris-dependents +GITHUB_TOKEN=$(gh auth token) cargo run --release -- scan \ + --owners hyperpolymath,metadatastician --json idris-dependents.json +cargo run -- parse-ipkg path/to/pkg.ipkg # offline: name + depends as JSON +---- + +== What it searches + +. *Catalogue* the `package` names of every `.ipkg` under `--owners` (or pass + `--packages`). +. Search `.ipkg` files for each package-family stem (`proven-dns`, + `proven-ffi`… → `proven`), and `pack.toml` / `.gitmodules` files mentioning + each owner. +. Classify each external repository: ++ +[cols="1,4"] +|=== +|HIGH |`pack.toml` or `.gitmodules` points at one of our GitHub owners +|MEDIUM |an `.ipkg` `depends` on a distinctive package name of ours +|LOW |only ambiguous names (`proven`, `trope`) with no corroboration +|=== + +Queries that GitHub truncates (over 1000 results) or marks incomplete are +listed in the report rather than silently dropped. + +== How the `.ipkg` reader is checked + +The lexer follows Idris2's own package lexer (`Parser/Lexer/Package.idr` and +`Common.idr` on Idris2 `main`): line comments, nested block comments that skip +strings, `==` as a single token, and any whitespace including CRLF. It is +lenient where Idris2 would reject a whole file: it skips a stray character +instead of failing. It also merges several `depends` fields, where Idris2 keeps +only the last. + +* `tests/rules_tests.rs`: the original JavaScript suite, plus regressions for + bugs it had. CRLF files and `==` bounds on continuation lines both lost + dependencies silently. +* `tests/roundtrip_props.rs`: a property test. Random well-formed `.ipkg` files + (CRLF, nested comments, bounds, both comma styles) are printed, parsed back + and checked. On 500 such files the original JavaScript parser was wrong 404 + times. +* `tests/oracle.rs`: a differential test against Idris2 itself. Set `IDRIS2` to + an `idris2` built from `main` (it needs `--dump-ipkg-json`, which 0.7.0 lacks). + The test is skipped when `IDRIS2` is unset. ++ +[source,bash] +---- +IDRIS2=/path/to/idris2 ORACLE_CASES=1000 cargo test --test oracle -- --nocapture +---- + +== Limits + +GitHub code search indexes default branches only and skips most forks. It +returns at most 1000 results per query. A package of ours that someone vendors +under a different name will not be found. diff --git a/idris-dependents/src/catalogue.rs b/idris-dependents/src/catalogue.rs new file mode 100644 index 00000000..dbfb749c --- /dev/null +++ b/idris-dependents/src/catalogue.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Which of your package names are worth searching for, and how confident a +//! match is. + +use serde::Serialize; + +/// Internal test/bench/ABI-shim packages nobody else would depend on by name. +pub fn is_distinctive(name: &str) -> bool { + let n = name.to_ascii_lowercase(); + let internal = matches!( + n.as_str(), + "abi" + | "benchmark" + | "benchmarks" + | "test" + | "tests" + | "coreonly" + | "ffi-test" + | "integration-tests" + | "property-tests" + ) || (n.starts_with("test-") && n.len() > 5) + || ["-test", "-tests", "-bench", "-suite"] + .iter() + .any(|suffix| n.len() > suffix.len() && n.ends_with(suffix)); + !internal +} + +/// Names that are ordinary words, so a bare match may be someone else's package. +pub const AMBIGUOUS: &[&str] = &["proven", "trope"]; + +/// One search term per package family: `proven-ffi`, `proven-dns`… collapse to +/// `proven`. Stems shorter than five characters are too noisy to search, so +/// those names are kept whole. +pub fn search_stems(names: &[String]) -> Vec { + let mut stems: Vec = names + .iter() + .filter(|n| is_distinctive(n)) + .map(|n| { + let stem = n.split('-').next().unwrap_or(n); + if stem.chars().count() >= 5 { + stem.to_string() + } else { + n.clone() + } + }) + .collect(); + stems.sort(); + stems.dedup(); + stems +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum Confidence { + /// `pack.toml` or `.gitmodules` points at your GitHub owners (unambiguous). + High, + /// `.ipkg` depends on a distinctive package name of yours. + Medium, + /// Only ambiguous names (`proven`, `trope`) with no corroboration. + Low, +} + +pub fn classify(deps: &[String], pack_refs: usize, sub_refs: usize) -> Option { + if pack_refs > 0 || sub_refs > 0 { + Some(Confidence::High) + } else if deps.iter().any(|d| !AMBIGUOUS.contains(&d.as_str())) { + Some(Confidence::Medium) + } else if !deps.is_empty() { + Some(Confidence::Low) + } else { + None + } +} diff --git a/idris-dependents/src/ipkg.rs b/idris-dependents/src/ipkg.rs new file mode 100644 index 00000000..5e18c4b8 --- /dev/null +++ b/idris-dependents/src/ipkg.rs @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: MPL-2.0 +//! `.ipkg` reading, modelled on Idris2's own package lexer and parser +//! (`src/Parser/Lexer/Package.idr`, `src/Parser/Lexer/Common.idr` and +//! `src/Idris/Package.idr` on Idris2 `main`). +//! +//! The lexer follows Idris2's token rules so that, on every file Idris2 +//! accepts, the dependency list agrees with `idris2 --dump-ipkg-json` +//! (checked by `tests/oracle.rs`). It is deliberately *lenient* where Idris2 +//! would reject the whole file: an unexpected character is skipped and an +//! unterminated comment or string runs to end of input. A search tool should +//! still see the `depends` of a file that has one stray character. + +/// One token of an `.ipkg` file. Comments and whitespace are dropped. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Tok { + /// An identifier; `dotted` when it has `.`-separated segments (a module + /// name such as `Proven.SafeMath`), which can never be a package name. + Ident { + text: String, + dotted: bool, + }, + Equals, + Sep, + Dot, + Lte, + Gte, + Lt, + Gt, + EqOp, + AndOp, + Str(String), + Int(String), +} + +fn is_ident_start(c: char) -> bool { + c == '_' || c.is_ascii_alphabetic() || c > '\u{a0}' +} + +fn is_ident_trailing(c: char) -> bool { + c == '-' || c == '\'' || c == '_' || c.is_ascii_alphanumeric() || c > '\u{a0}' +} + +/// Tokenise `src`. Returns the tokens and the number of characters skipped +/// because no Idris2 token rule matches them. +pub fn lex(src: &str) -> (Vec, usize) { + let s: Vec = src.chars().collect(); + let at = |j: usize| s.get(j).copied(); + let mut toks = Vec::new(); + let mut skipped = 0; + let mut i = 0; + while let Some(c) = at(i) { + // Line comment: two or more dashes not followed by `}`. + if c == '-' && at(i + 1) == Some('-') { + let mut j = i + 2; + while at(j) == Some('-') { + j += 1; + } + if at(j) != Some('}') { + while at(j).is_some_and(|c| c != '\n') { + j += 1; + } + i = j; + continue; + } + } + if c == '{' && at(i + 1) == Some('-') { + i = block_comment(&s, i); + continue; + } + if c.is_whitespace() { + i += 1; + continue; + } + if is_ident_start(c) { + let mut j = i + 1; + while at(j).is_some_and(is_ident_trailing) { + j += 1; + } + let mut dotted = false; + while at(j) == Some('.') && at(j + 1).is_some_and(is_ident_start) { + dotted = true; + j += 2; + while at(j).is_some_and(is_ident_trailing) { + j += 1; + } + } + toks.push(Tok::Ident { + text: s[i..j].iter().collect(), + dotted, + }); + i = j; + continue; + } + if c.is_ascii_digit() { + let mut j = i + 1; + while at(j).is_some_and(|c| c.is_ascii_digit()) { + j += 1; + } + toks.push(Tok::Int(s[i..j].iter().collect())); + i = j; + continue; + } + if c == '"' { + if let Some(end) = string_end(&s, i) { + toks.push(Tok::Str(s[i + 1..end - 1].iter().collect())); + i = end; + } else { + skipped += 1; + i += 1; + } + continue; + } + let (tok, len) = match (c, at(i + 1)) { + ('<', Some('=')) => (Tok::Lte, 2), + ('>', Some('=')) => (Tok::Gte, 2), + ('=', Some('=')) => (Tok::EqOp, 2), + ('&', Some('&')) => (Tok::AndOp, 2), + ('<', _) => (Tok::Lt, 1), + ('>', _) => (Tok::Gt, 1), + ('=', _) => (Tok::Equals, 1), + (',', _) => (Tok::Sep, 1), + ('.', _) => (Tok::Dot, 1), + _ => { + skipped += 1; + i += 1; + continue; + } + }; + toks.push(tok); + i += len; + } + (toks, skipped) +} + +/// Index just past the closing quote of the string literal opening at `i`. +fn string_end(s: &[char], i: usize) -> Option { + let mut j = i + 1; + while j < s.len() { + match s[j] { + '\\' => j += 2, + '"' => return Some(j + 1), + _ => j += 1, + } + } + None +} + +/// Skip a block comment opening at `i` (`{-`). Mirrors Idris2's +/// `toEndComment` / `singleBrace` / `singleDash` / `doubleDash` automaton: +/// comments nest, string and character literals inside them are skipped, +/// and `--` inside a comment hides the rest of its line (including any `-}`). +fn block_comment(s: &[char], i: usize) -> usize { + let at = |j: usize| s.get(j).copied(); + let mut j = i + 2; + while at(j) == Some('-') { + j += 1; + } + let mut depth = 1usize; + // `dash`: the previous character was a dash that may start `-}` or `--`. + let mut dash = false; + loop { + let Some(c) = at(j) else { return s.len() }; + if dash { + dash = false; + match c { + '-' => { + while at(j) == Some('-') { + j += 1; + } + if at(j) == Some('}') { + j += 1; + depth -= 1; + } else { + while at(j).is_some_and(|c| c != '\n') { + j += 1; + } + } + } + '}' => { + j += 1; + depth -= 1; + } + _ => {} + } + if depth == 0 { + return j; + } + continue; + } + match c { + '{' if at(j + 1) == Some('-') => { + j += 2; + while at(j) == Some('-') { + j += 1; + } + depth += 1; + dash = true; + } + '-' => { + j += 1; + dash = true; + } + '"' => j = string_end(s, j).unwrap_or(j + 1), + '\'' => { + j += match (at(j + 1), at(j + 2), at(j + 3)) { + (Some('\\'), Some(_), Some('\'')) => 4, + (Some(x), Some('\''), _) if x != '\'' => 3, + _ => 1, + } + } + _ => j += 1, + } + } +} + +fn ident(t: Option<&Tok>) -> Option<&str> { + match t { + Some(Tok::Ident { + text, + dotted: false, + }) => Some(text), + _ => None, + } +} + +/// Package name from the `package ` declaration, or `None`. +pub fn package_name(text: &str) -> Option { + let (toks, _) = lex(text); + (0..toks.len()).find_map(|i| { + match (ident(toks.get(i)), ident(toks.get(i + 1)), toks.get(i + 2)) { + (Some("package"), Some(name), next) if next != Some(&Tok::Equals) => { + Some(name.to_string()) + } + _ => None, + } + }) +} + +/// Package names in every `depends =` field, in order, without duplicates. +/// +/// A field key is an identifier followed by a single `=`; version bounds use +/// `==`, `>=` and friends, which lex as different tokens, so a bound can +/// never be mistaken for the start of the next field. +pub fn depends(text: &str) -> Vec { + let (toks, _) = lex(text); + let mut out: Vec = Vec::new(); + let mut i = 0; + while i + 1 < toks.len() { + if ident(toks.get(i)) != Some("depends") || toks[i + 1] != Tok::Equals { + i += 1; + continue; + } + let mut j = i + 2; + while let Some(name) = ident(toks.get(j)) { + if toks.get(j + 1) == Some(&Tok::Equals) { + break; // a trailing comma ran into the next field's key + } + if !out.iter().any(|d| d == name) { + out.push(name.to_string()); + } + j += 1; + while matches!( + toks.get(j), + Some( + Tok::Lt + | Tok::Lte + | Tok::Gt + | Tok::Gte + | Tok::EqOp + | Tok::AndOp + | Tok::Int(_) + | Tok::Dot + ) + ) { + j += 1; + } + if toks.get(j) != Some(&Tok::Sep) { + break; + } + j += 1; + } + i = j.max(i + 1); + } + out +} diff --git a/idris-dependents/src/lib.rs b/idris-dependents/src/lib.rs new file mode 100644 index 00000000..e3f3fe03 --- /dev/null +++ b/idris-dependents/src/lib.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: MPL-2.0 +//! idris-dependents — find people outside your own accounts whose Idris2 +//! projects depend on your packages, using GitHub code search (GET only). +//! +//! The parsing and classification rules are pure functions so they can be +//! tested without the network; [`scan::run`] does the search walk through the +//! [`scan::Io`] trait. + +pub mod catalogue; +pub mod ipkg; +pub mod refs; +pub mod scan; diff --git a/idris-dependents/src/main.rs b/idris-dependents/src/main.rs new file mode 100644 index 00000000..f28e3cd8 --- /dev/null +++ b/idris-dependents/src/main.rs @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: MPL-2.0 +//! CLI for idris-dependents. +//! +//! GITHUB_TOKEN=$(gh auth token) idris-dependents scan \ +//! --owners hyperpolymath,metadatastician --json idris-dependents.json + +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result}; +use clap::{Parser, Subcommand}; + +use idris_dependents::scan::{self, Io, Options, Report, Response}; +use idris_dependents::{ipkg, refs}; + +#[derive(Parser)] +#[command( + version, + about = "Find external Idris2 projects that depend on your packages (GitHub code search, GET only)" +)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Search GitHub. Needs GITHUB_TOKEN or GH_TOKEN (code search requires auth). + Scan { + /// GitHub owners whose packages to look for. + #[arg( + long, + value_delimiter = ',', + default_value = "hyperpolymath,metadatastician" + )] + owners: Vec, + /// Your own accounts, never reported (defaults to --owners). + #[arg(long = "self", value_delimiter = ',')] + self_accounts: Option>, + /// Use these package names instead of cataloguing your .ipkg files. + #[arg(long, value_delimiter = ',')] + packages: Option>, + /// Also write the report as JSON to this path. + #[arg(long)] + json: Option, + }, + /// Print the package name and `depends` of a local .ipkg as JSON (offline). + ParseIpkg { file: PathBuf }, + /// Print the pack.toml custom entries pointing at --owners as JSON (offline). + ParsePackToml { + file: PathBuf, + #[arg( + long, + value_delimiter = ',', + default_value = "hyperpolymath,metadatastician" + )] + owners: Vec, + }, +} + +struct Live { + client: reqwest::blocking::Client, + token: String, +} + +impl Io for Live { + fn get(&mut self, url: &str, raw: bool) -> Result { + let accept = if raw { + "application/vnd.github.raw" + } else { + "application/vnd.github+json" + }; + let res = self + .client + .get(url) + .bearer_auth(&self.token) + .header("Accept", accept) + .header("X-GitHub-Api-Version", "2022-11-28") + .header("User-Agent", "idris-dependents") + .send() + .with_context(|| format!("GET {url}"))?; + let status = res.status().as_u16(); + let headers: BTreeMap = res + .headers() + .iter() + .filter_map(|(k, v)| { + Some(( + k.as_str().to_ascii_lowercase(), + v.to_str().ok()?.to_string(), + )) + }) + .collect(); + Ok(Response { + status, + headers, + body: res.text()?, + }) + } + fn sleep(&mut self, d: Duration) { + std::thread::sleep(d); + } + fn now_secs(&self) -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) + } + fn log(&mut self, msg: &str) { + eprintln!("{msg}"); + } +} + +fn print_report(r: &Report) { + let owners: std::collections::BTreeSet<&str> = + r.rows.iter().map(|x| x.owner.as_str()).collect(); + println!( + "\nChecked {} package names. {} external repo(s), {} owner(s).\n", + r.catalogue_size, + r.rows.len(), + owners.len() + ); + for row in &r.rows { + let mut what = Vec::new(); + if !row.deps.is_empty() { + what.push(format!("depends: {}", row.deps.join(", "))); + } + if !row.pack_refs.is_empty() { + let names: Vec<&str> = row.pack_refs.iter().map(|p| p.name.as_str()).collect(); + what.push(format!("pack: {}", names.join(", "))); + } + if !row.sub_refs.is_empty() { + what.push(format!("submodule: {}", row.sub_refs.join(", "))); + } + let conf = serde_json::to_value(row.confidence) + .ok() + .and_then(|v| v.as_str().map(str::to_string)) + .unwrap_or_default(); + println!("[{conf}] {} — {}", row.repo, what.join(" | ")); + for f in &row.files { + println!(" {f}"); + } + } + if r.rows.is_empty() { + println!("No external Idris2 dependents found in GitHub's code-search index."); + } + if !r.incomplete_queries.is_empty() { + println!( + "\nIncomplete (GitHub truncated or timed out): {}", + r.incomplete_queries.join("; ") + ); + } +} + +fn main() -> Result<()> { + match Cli::parse().command { + Command::ParseIpkg { file } => { + let text = std::fs::read_to_string(&file) + .with_context(|| format!("reading {}", file.display()))?; + let out = serde_json::json!({ "package": ipkg::package_name(&text), "depends": ipkg::depends(&text) }); + println!("{out}"); + } + Command::ParsePackToml { file, owners } => { + let text = std::fs::read_to_string(&file) + .with_context(|| format!("reading {}", file.display()))?; + println!( + "{}", + serde_json::to_string(&refs::pack_toml_refs(&text, &owners)?)? + ); + } + Command::Scan { + owners, + self_accounts, + packages, + json, + } => { + let token = std::env::var("GITHUB_TOKEN") + .or_else(|_| std::env::var("GH_TOKEN")) + .ok() + .filter(|t| !t.is_empty()) + .context("set GITHUB_TOKEN (e.g. GITHUB_TOKEN=$(gh auth token)); code search requires auth")?; + let self_accounts = self_accounts.unwrap_or_else(|| owners.clone()); + let opts = Options { + owners, + self_accounts, + packages, + }; + let mut io = Live { + client: reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(60)) + .build()?, + token, + }; + let report = scan::run(&opts, &mut io)?; + print_report(&report); + if let Some(path) = json { + std::fs::write(&path, serde_json::to_string_pretty(&report)?)?; + eprintln!("wrote {}", path.display()); + } + } + } + Ok(()) +} diff --git a/idris-dependents/src/refs.rs b/idris-dependents/src/refs.rs new file mode 100644 index 00000000..7d00e14c --- /dev/null +++ b/idris-dependents/src/refs.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: MPL-2.0 +//! References to GitHub owners from `pack.toml` and `.gitmodules`. + +use serde::Serialize; + +/// A `[custom..]` entry in `pack.toml` whose `url` points at +/// one of the watched owners. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct PackRef { + pub name: String, + pub url: String, +} + +/// True when `url` is a GitHub URL (https, ssh or scp form) under one of +/// `owners`. The owner must match exactly (case-insensitive), and `github.com` +/// must be the whole host, so `notgithub.com/owner/x` does not count. +pub fn points_at(url: &str, owners: &[String]) -> bool { + let url = url.to_ascii_lowercase(); + let mut from = 0; + while let Some(k) = url[from..].find("github.com") { + let at = from + k; + let host_start = at == 0 || matches!(url.as_bytes()[at - 1], b'/' | b'@' | b'.'); + let rest = &url[at + "github.com".len()..]; + if host_start && (rest.starts_with('/') || rest.starts_with(':')) { + let path = &rest[1..]; + if owners.iter().any(|o| { + let o = o.to_ascii_lowercase(); + path.strip_prefix(o.as_str()) + .is_some_and(|r| r.starts_with('/')) + }) { + return true; + } + } + from = at + 1; + } + false +} + +/// `pack.toml` custom entries pointing at `owners`, in file order. +/// Uses a real TOML parser, so quoting styles and `#` inside strings are +/// handled the way pack itself handles them. +pub fn pack_toml_refs(text: &str, owners: &[String]) -> Result, toml::de::Error> { + let doc: toml::Table = text.parse()?; + let mut refs = Vec::new(); + let Some(custom) = doc.get("custom").and_then(|v| v.as_table()) else { + return Ok(refs); + }; + for collection in custom.values().filter_map(|v| v.as_table()) { + for (name, entry) in collection { + if let Some(url) = entry.get("url").and_then(|u| u.as_str()) { + if points_at(url, owners) { + refs.push(PackRef { + name: name.clone(), + url: url.to_string(), + }); + } + } + } + } + Ok(refs) +} + +/// Submodule URLs in `.gitmodules` (git-config syntax) pointing at `owners`. +pub fn gitmodules_refs(text: &str, owners: &[String]) -> Vec { + text.lines() + .filter_map(|line| { + let line = line.trim(); + let value = line + .strip_prefix("url")? + .trim_start() + .strip_prefix('=')? + .trim(); + let value = match value.strip_prefix('"') { + Some(q) => q.split('"').next().unwrap_or(""), + None => value.split(['#', ';']).next().unwrap_or("").trim(), + }; + points_at(value, owners).then(|| value.to_string()) + }) + .collect() +} diff --git a/idris-dependents/src/scan.rs b/idris-dependents/src/scan.rs new file mode 100644 index 00000000..003c187e --- /dev/null +++ b/idris-dependents/src/scan.rs @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: MPL-2.0 +//! The GitHub code-search walk. All I/O goes through [`Io`], whose only +//! network verb is `get`: the audit cannot write to GitHub by construction. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::time::Duration; + +use anyhow::{bail, Context, Result}; +use serde::Serialize; + +use crate::catalogue::{classify, is_distinctive, search_stems, Confidence}; +use crate::{ipkg, refs}; + +pub const API: &str = "https://api.github.com"; +/// Authenticated code search allows 10 requests a minute. +pub const SEARCH_GAP: Duration = Duration::from_millis(6500); +/// GitHub code search never returns more than this many results per query. +const SEARCH_CAP: u64 = 1000; + +pub struct Response { + pub status: u16, + /// Header names in lower case. + pub headers: BTreeMap, + pub body: String, +} + +/// Side effects the scan needs. `raw` asks for file contents instead of JSON. +pub trait Io { + fn get(&mut self, url: &str, raw: bool) -> Result; + fn sleep(&mut self, d: Duration); + fn now_secs(&self) -> u64; + fn log(&mut self, msg: &str); +} + +pub struct Options { + pub owners: Vec, + /// Accounts whose repositories are yours and are never reported. + pub self_accounts: Vec, + /// Skip cataloguing and use these package names. + pub packages: Option>, +} + +#[derive(Debug, Serialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct Row { + pub owner: String, + pub repo: String, + pub deps: Vec, + pub pack_refs: Vec, + pub sub_refs: Vec, + pub files: Vec, + pub confidence: Confidence, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Report { + pub catalogue_size: usize, + pub rows: Vec, + /// Queries whose results GitHub truncated or marked incomplete. + pub incomplete_queries: Vec, +} + +#[derive(Clone, Copy, PartialEq)] +enum Kind { + Ipkg, + Pack, + Submodule, +} + +struct Hit { + url: String, + html_url: String, + full_name: String, + owner: String, +} + +#[derive(Default)] +struct Acc { + owner: String, + deps: BTreeSet, + pack_refs: Vec, + sub_refs: Vec, + files: Vec, +} + +struct Scanner<'a, I: Io> { + io: &'a mut I, + incomplete: Vec, +} + +impl Scanner<'_, I> { + fn get(&mut self, url: &str, raw: bool) -> Result { + for _ in 0..=5 { + let res = self.io.get(url, raw)?; + let h = |k: &str| res.headers.get(k).map(String::as_str); + let limited = res.status == 429 + || (res.status == 403 + && (h("retry-after").is_some() || h("x-ratelimit-remaining") == Some("0"))); + if !limited { + if !(200..300).contains(&res.status) { + bail!("HTTP {} for {url}", res.status); + } + return Ok(res.body); + } + let retry_after = h("retry-after").and_then(|v| v.parse::().ok()); + let reset = h("x-ratelimit-reset").and_then(|v| v.parse::().ok()); + let wait = match (retry_after, reset) { + (Some(s), _) if s > 0 => s, + (_, Some(r)) => r.saturating_sub(self.io.now_secs()).max(1) + 1, + _ => 60, + }; + self.io.log(&format!("rate-limited; waiting {wait}s")); + self.io.sleep(Duration::from_secs(wait)); + } + bail!("still rate-limited after 5 retries: {url}") + } + + fn search_all(&mut self, q: &str) -> Result> { + let mut items = Vec::new(); + for page in 1..=10 { + let url = format!( + "{API}/search/code?q={}&per_page=100&page={page}", + urlencode(q) + ); + let body: serde_json::Value = serde_json::from_str(&self.get(&url, false)?) + .with_context(|| format!("search response for {q}"))?; + self.io.sleep(SEARCH_GAP); + let total = body["total_count"].as_u64().unwrap_or(0); + if page == 1 && total > SEARCH_CAP { + self.io.log(&format!( + " note: {total} results for `{q}`; GitHub returns at most {SEARCH_CAP}" + )); + self.incomplete.push(q.to_string()); + } + if body["incomplete_results"].as_bool() == Some(true) { + self.io.log(&format!( + " note: GitHub marked results incomplete for: {q}" + )); + if !self.incomplete.iter().any(|x| x == q) { + self.incomplete.push(q.to_string()); + } + } + let page_items = body["items"].as_array().cloned().unwrap_or_default(); + let n = page_items.len(); + items.extend(page_items.iter().filter_map(|it| { + Some(Hit { + url: it["url"].as_str()?.to_string(), + html_url: it["html_url"].as_str()?.to_string(), + full_name: it["repository"]["full_name"].as_str()?.to_string(), + owner: it["repository"]["owner"]["login"].as_str()?.to_string(), + }) + })); + if n < 100 || items.len() as u64 >= total { + break; + } + } + Ok(items) + } +} + +/// Percent-encode a query string component (RFC 3986 unreserved kept). +fn urlencode(s: &str) -> String { + s.bytes() + .map(|b| match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + (b as char).to_string() + } + _ => format!("%{b:02X}"), + }) + .collect() +} + +pub fn run(opts: &Options, io: &mut I) -> Result { + let self_set: HashSet = opts + .self_accounts + .iter() + .map(|s| s.to_ascii_lowercase()) + .collect(); + let mut sc = Scanner { + io, + incomplete: Vec::new(), + }; + + // 1. Catalogue of your package names (or use --packages to skip this step). + let catalogue = match &opts.packages { + Some(p) => p.clone(), + None => { + let mut names = BTreeSet::new(); + for o in &opts.owners { + sc.io.log(&format!("cataloguing .ipkg packages under {o}…")); + for it in sc.search_all(&format!("package extension:ipkg user:{o}"))? { + if let Some(n) = ipkg::package_name(&sc.get(&it.url, true)?) { + names.insert(n); + } + } + } + names.into_iter().collect() + } + }; + let distinctive: HashSet<&str> = catalogue + .iter() + .map(String::as_str) + .filter(|n| is_distinctive(n)) + .collect(); + let stems = search_stems(&catalogue); + sc.io.log(&format!( + "{} distinctive package names; stems: {}", + distinctive.len(), + stems.join(", ") + )); + + // 2. Package-name stems in .ipkg files, plus pack.toml / .gitmodules pointing at you. + let mut queries: Vec<(String, Kind)> = stems + .iter() + .map(|s| (format!("{s} extension:ipkg"), Kind::Ipkg)) + .collect(); + for o in &opts.owners { + queries.push((format!("{o} filename:pack.toml"), Kind::Pack)); + queries.push((format!("{o} filename:.gitmodules"), Kind::Submodule)); + } + + let mut hits: BTreeMap = BTreeMap::new(); + let mut seen_files = HashSet::new(); + for (q, kind) in &queries { + sc.io.log(&format!("search: {q}")); + for it in sc.search_all(q)? { + if self_set.contains(&it.owner.to_ascii_lowercase()) + || !seen_files.insert(it.html_url.clone()) + { + continue; + } + let text = sc.get(&it.url, true)?; + let acc = hits.entry(it.full_name.clone()).or_default(); + acc.owner = it.owner.clone(); + let found = match kind { + Kind::Ipkg => { + let before = acc.deps.len(); + acc.deps.extend( + ipkg::depends(&text) + .into_iter() + .filter(|d| distinctive.contains(d.as_str())), + ); + acc.deps.len() > before + } + Kind::Pack => match refs::pack_toml_refs(&text, &opts.owners) { + Ok(r) => { + let found = !r.is_empty(); + acc.pack_refs.extend(r); + found + } + Err(e) => { + sc.io.log(&format!( + " skipping unparseable pack.toml {}: {e}", + it.html_url + )); + false + } + }, + Kind::Submodule => { + let r = refs::gitmodules_refs(&text, &opts.owners); + let found = !r.is_empty(); + acc.sub_refs.extend(r); + found + } + }; + if found { + acc.files.push(it.html_url); + } + } + } + + // 3. Classify; most confident first, then by repository name. + let mut rows: Vec = hits + .into_iter() + .filter(|(_, a)| !a.files.is_empty()) + .filter_map(|(repo, a)| { + let deps: Vec = a.deps.into_iter().collect(); + let confidence = classify(&deps, a.pack_refs.len(), a.sub_refs.len())?; + Some(Row { + owner: a.owner, + repo, + deps, + pack_refs: a.pack_refs, + sub_refs: a.sub_refs, + files: a.files, + confidence, + }) + }) + .collect(); + rows.sort_by(|a, b| { + a.confidence + .cmp(&b.confidence) + .then_with(|| a.repo.cmp(&b.repo)) + }); + Ok(Report { + catalogue_size: distinctive.len(), + rows, + incomplete_queries: sc.incomplete, + }) +} diff --git a/idris-dependents/tests/common/mod.rs b/idris-dependents/tests/common/mod.rs new file mode 100644 index 00000000..58b56c57 --- /dev/null +++ b/idris-dependents/tests/common/mod.rs @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Random `.ipkg` files that Idris2 itself accepts, with their expected +//! `depends`. Shared by the round-trip property test and the Idris2 oracle. +//! +//! Only well-formed files are generated (one `depends` field, at most one +//! lower and one upper bound per dependency, as `mkBound` in Idris2 requires), +//! so the oracle never rejects a case for a reason unrelated to the parser. + +#![allow(dead_code)] + +use proptest::prelude::*; + +#[derive(Debug, Clone)] +pub struct Ipkg { + pub text: String, + pub name: String, + pub deps: Vec, +} + +fn pkg_name() -> impl Strategy { + "[a-z][a-z0-9_']{0,6}(-[a-z0-9]{1,4}){0,2}" +} + +fn version() -> impl Strategy { + prop::collection::vec(0u32..20, 1..4) + .prop_map(|v| v.iter().map(u32::to_string).collect::>().join(".")) +} + +fn bounds() -> impl Strategy { + prop_oneof![ + Just(String::new()), + (prop::sample::select(vec![">", ">="]), version()).prop_map(|(o, v)| format!(" {o} {v}")), + (prop::sample::select(vec!["<", "<="]), version()).prop_map(|(o, v)| format!(" {o} {v}")), + (version(), version()).prop_map(|(a, b)| format!(" >= {a} && < {b}")), + (version(), version()).prop_map(|(a, b)| format!(" <= {b} && > {a}")), + version().prop_map(|v| format!(" == {v}")), + ] +} + +/// Line ending: LF or CRLF. +fn nl() -> impl Strategy { + prop::sample::select(vec!["\n", "\r\n"]) +} + +/// Text that is safe inside a string or a line comment and full of +/// comment-like noise (`{-`, `-}`, `depends = fake`). +fn noisy_text() -> impl Strategy { + prop::collection::vec( + prop::sample::select(vec![ + "a", + " ", + "{-", + "-}", + "--", + "depends = fake", + ",", + "x1", + "}", + ]), + 0..6, + ) + .prop_map(|v| v.concat()) +} + +/// A block comment, possibly nested, containing strings and line comments +/// whose `-}` must not close it. +fn block_comment() -> impl Strategy { + let leaf = prop_oneof![ + "[a-z ,=.]{0,8}".prop_map(|s| s), + noisy_text().prop_map(|t| format!("\"{t}\"")), + (noisy_text(), nl()).prop_map(|(t, n)| format!("-- {t}{n}")), + ]; + leaf.prop_recursive(3, 12, 4, |inner| { + prop::collection::vec(inner, 0..4).prop_map(|segs| format!("{{- {} -}}", segs.join(" "))) + }) + .prop_map(|s| { + if s.starts_with("{-") { + s + } else { + format!("{{- {s} -}}") + } + }) +} + +/// Whitespace, optionally carrying a comment, between two tokens. +fn gap() -> impl Strategy { + prop_oneof![ + 4 => prop::sample::select(vec![" ", " ", "\t", " \n ", " \r\n\t"]).prop_map(String::from), + 1 => (noisy_text(), nl()).prop_map(|(t, n)| format!(" -- {t}{n} ")), + 1 => block_comment().prop_map(|c| format!(" {c} ")), + ] +} + +fn module_name() -> impl Strategy { + "[A-Z][a-z]{0,5}(\\.[A-Z][a-z]{0,5}){0,2}" +} + +pub fn ipkg() -> impl Strategy { + ( + pkg_name(), + prop::collection::vec((pkg_name(), bounds()), 0..6), + any::(), // leading-comma style + prop::collection::vec(gap(), 16), + prop::collection::vec(module_name(), 1..4), + noisy_text(), + nl(), + any::(), // depends before or after the other fields + ) + .prop_map( + |(name, raw_deps, leading, gaps, modules, opts, nl, deps_first)| { + let mut deps: Vec<(String, String)> = Vec::new(); + for (d, b) in raw_deps { + if !deps.iter().any(|(x, _)| *x == d) { + deps.push((d, b)); + } + } + let g = |i: usize| gaps[i % gaps.len()].clone(); + let mut depends_field = String::new(); + if !deps.is_empty() { + depends_field.push_str(&format!("depends{}={}", g(0), g(1))); + for (i, (d, b)) in deps.iter().enumerate() { + if i > 0 { + if leading { + depends_field.push_str(&format!("{nl}{}, ", g(i + 2))); + } else { + depends_field.push_str(&format!(",{}", g(i + 2))); + } + } + depends_field.push_str(&format!("{d}{b}")); + } + depends_field.push_str(nl); + } + let others = format!( + "version = 1.2.3{nl}opts = \"{opts}\"{nl}modules ={}{}{nl}sourcedir = \"src\"{nl}", + g(10), + modules.join(&format!("{},{}", g(11), g(12))) + ); + let body = if deps_first { + format!("{depends_field}{others}") + } else { + format!("{others}{depends_field}") + }; + let text = format!( + "{}package {name}{}{nl}{body}{}", + g(13).trim_start(), + g(14), + g(15) + ); + Ipkg { + text, + name, + deps: deps.into_iter().map(|(d, _)| d).collect(), + } + }, + ) +} diff --git a/idris-dependents/tests/dump_samples.rs b/idris-dependents/tests/dump_samples.rs new file mode 100644 index 00000000..cbb92d54 --- /dev/null +++ b/idris-dependents/tests/dump_samples.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Writes generated cases to $DUMP_IPKG_DIR (used to check the generator's power +//! and to feed the Idris2 oracle by hand). Does nothing when the variable is unset. + +mod common; + +use proptest::strategy::{Strategy, ValueTree}; +use proptest::test_runner::TestRunner; + +#[test] +fn dump_samples() { + let Ok(dir) = std::env::var("DUMP_IPKG_DIR") else { + return; + }; + let n: usize = std::env::var("DUMP_IPKG_N") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(300); + std::fs::create_dir_all(&dir).unwrap(); + let mut runner = TestRunner::deterministic(); + for i in 0..n { + let p = common::ipkg().new_tree(&mut runner).unwrap().current(); + std::fs::write(format!("{dir}/case{i:04}.ipkg"), &p.text).unwrap(); + std::fs::write( + format!("{dir}/case{i:04}.json"), + serde_json::json!({"package": p.name, "depends": p.deps}).to_string(), + ) + .unwrap(); + } +} diff --git a/idris-dependents/tests/fixtures/ochrance.ipkg b/idris-dependents/tests/fixtures/ochrance.ipkg new file mode 100644 index 00000000..5d23cb75 --- /dev/null +++ b/idris-dependents/tests/fixtures/ochrance.ipkg @@ -0,0 +1,32 @@ +-- SPDX-License-Identifier: MPL-2.0 +-- Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) +-- +-- Ochrance Framework - Verified Subsystem Integrity Framework +-- Package definition for Idris2 + +package ochrance + +version = "0.1.0" +authors = "Jonathan D.A. Jewell" +brief = "Ochránce: Verified Subsystem Integrity Framework" +license = "MPL-2.0" + +sourcedir = "ochrance-core" + +depends = base, + contrib, + network + +modules = Ochrance.Framework.Interface, + Ochrance.Framework.Proof, + Ochrance.Framework.Error, + Ochrance.Framework.Progressive, + Ochrance.A2ML.Types, + Ochrance.A2ML.Lexer, + Ochrance.A2ML.Parser, + Ochrance.A2ML.Validator, + Ochrance.A2ML.Serializer, + Ochrance.Filesystem.Types, + Ochrance.Filesystem.Merkle, + Ochrance.Filesystem.Verify, + Ochrance.Filesystem.Repair diff --git a/idris-dependents/tests/fixtures/ordering-laws.ipkg b/idris-dependents/tests/fixtures/ordering-laws.ipkg new file mode 100644 index 00000000..b65e9cbc --- /dev/null +++ b/idris-dependents/tests/fixtures/ordering-laws.ipkg @@ -0,0 +1,15 @@ +package proven-ordering-laws + +version = 0.2.0 +sourcedir = "src" + +main = ProvenSubject.OrderingLawsMain +executable = "proven-ordering-laws" + +-- proven-safeordering is proven's Proven.SafeOrdering at PROVEN_PIN, installed +-- by scripts/prove-proven-ordering.sh; it is not a package proven itself ships. +depends = proven-tests, proven-safeordering + +modules = ProvenSubject.OrderingLaws, + ProvenSubject.OrderingLawsTests, + ProvenSubject.OrderingLawsMain diff --git a/idris-dependents/tests/fixtures/pack.toml b/idris-dependents/tests/fixtures/pack.toml new file mode 100644 index 00000000..baabef86 --- /dev/null +++ b/idris-dependents/tests/fixtures/pack.toml @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) +# pack.toml - Pack package manager configuration for proven + +[package] +name = "proven" +version = "1.1.0" +authors = ["Jonathan D.A. Jewell (hyperpolymath) "] +brief = "Verified safe operations - code that cannot crash" +langversion = ">= 0.6.0" + +[build] +ipkg = "proven.ipkg" + +[custom.all.proven] +type = "local" +path = "." +ipkg = "proven.ipkg" diff --git a/idris-dependents/tests/fixtures/proven.ipkg b/idris-dependents/tests/fixtures/proven.ipkg new file mode 100644 index 00000000..6456dbec --- /dev/null +++ b/idris-dependents/tests/fixtures/proven.ipkg @@ -0,0 +1,378 @@ +-- SPDX-License-Identifier: MPL-2.0 +-- Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) +package proven + +version = 1.1.0 +authors = "Jonathan D.A. Jewell (hyperpolymath) " +license = "MPL-2.0" +brief = "Safe-operations library with proof companions (see PROOF-NEEDS.md for discharge status)" +readme = "README.adoc" +homepage = "https://github.com/hyperpolymath/proven" +sourceloc = "https://github.com/hyperpolymath/proven" +bugtracker = "https://github.com/hyperpolymath/proven/issues" + +sourcedir = "src" +builddir = "build" +outputdir = "build/exec" + +modules = Proven + , Proven.Core + -- SafeMath + , Proven.SafeMath + , Proven.SafeMath.Nat + , Proven.SafeMath.Int + , Proven.SafeMath.Bounded + , Proven.SafeMath.Proofs + -- SafeString + , Proven.SafeString + , Proven.SafeString.UTF8 + , Proven.SafeString.Escape + , Proven.SafeString.Proofs + -- SafeJson + , Proven.SafeJson + , Proven.SafeJson.Parser + , Proven.SafeJson.Access + , Proven.SafeJson.Proofs + -- SafeUrl + , Proven.SafeUrl + , Proven.SafeUrl.Parser + , Proven.SafeUrl.Query + , Proven.SafeUrl.Proofs + -- SafeEmail + , Proven.SafeEmail + , Proven.SafeEmail.Parser + , Proven.SafeEmail.Validation + , Proven.SafeEmail.Proofs + -- SafePath + , Proven.SafePath + , Proven.SafePath.Types + , Proven.SafePath.Operations + , Proven.SafePath.Proofs + -- SafeCrypto + , Proven.SafeCrypto + , Proven.SafeCrypto.Hash + , Proven.SafeCrypto.Random + , Proven.SafeCrypto.Proofs + -- SafeInput + , Proven.SafeInput + -- SafePassword + , Proven.SafePassword + , Proven.SafePassword.Policy + , Proven.SafePassword.Hash + , Proven.SafePassword.Strength + , Proven.SafePassword.Proofs + -- SafeDateTime + , Proven.SafeDateTime + , Proven.SafeDateTime.Types + , Proven.SafeDateTime.Parse + , Proven.SafeDateTime.Zones + , Proven.SafeDateTime.Proofs + -- SafeNetwork + , Proven.SafeNetwork + , Proven.SafeNetwork.IPv4 + , Proven.SafeNetwork.IPv6 + , Proven.SafeNetwork.CIDR + , Proven.SafeNetwork.Port + , Proven.SafeNetwork.Proofs + -- SafeHtml + , Proven.SafeHtml + , Proven.SafeHtml.Escape + , Proven.SafeHtml.Builder + , Proven.SafeHtml.Sanitize + , Proven.SafeHtml.Proofs + -- SafeRegex + , Proven.SafeRegex + , Proven.SafeRegex.Types + , Proven.SafeRegex.Safety + , Proven.SafeRegex.Proofs + -- SafeCommand + , Proven.SafeCommand + , Proven.SafeCommand.Escape + , Proven.SafeCommand.Builder + , Proven.SafeCommand.Proofs + -- SafeSQL + , Proven.SafeSQL + , Proven.SafeSQL.Types + , Proven.SafeSQL.Params + , Proven.SafeSQL.Proofs + -- SafeJWT + , Proven.SafeJWT + , Proven.SafeJWT.Decode + , Proven.SafeJWT.Validate + , Proven.SafeJWT.Proofs + -- SafeBase64 + , Proven.SafeBase64 + , Proven.SafeBase64.Types + , Proven.SafeBase64.Encode + , Proven.SafeBase64.Decode + , Proven.SafeBase64.Proofs + -- SafeXML + , Proven.SafeXML + , Proven.SafeXML.Types + , Proven.SafeXML.Builder + , Proven.SafeXML.Proofs + -- SafeYAML + , Proven.SafeYAML + , Proven.SafeYAML.Types + , Proven.SafeYAML.Proofs + -- SafeTOML + , Proven.SafeTOML + , Proven.SafeTOML.Types + , Proven.SafeTOML.Parser + , Proven.SafeTOML.Proofs + -- SafeEnv + , Proven.SafeEnv + , Proven.SafeEnv.Types + , Proven.SafeEnv.Access + , Proven.SafeEnv.Proofs + -- SafeArgs + , Proven.SafeArgs + , Proven.SafeArgs.Types + , Proven.SafeArgs.Parser + , Proven.SafeArgs.Proofs + -- SafeFile + , Proven.SafeFile + , Proven.SafeFile.Types + , Proven.SafeFile.Operations + , Proven.SafeFile.Proofs + -- SafeHeader + , Proven.SafeHeader + , Proven.SafeHeader.Types + , Proven.SafeHeader.Parser + , Proven.SafeHeader.Proofs + -- SafeCookie + , Proven.SafeCookie + , Proven.SafeCookie.Types + , Proven.SafeCookie.Parser + , Proven.SafeCookie.Proofs + -- SafeContentType + , Proven.SafeContentType + , Proven.SafeContentType.Types + , Proven.SafeContentType.Parser + , Proven.SafeContentType.Proofs + -- Standalone validated types + , Proven.SafeUUID + , Proven.SafeCurrency + , Proven.SafePhone + , Proven.SafeHex + -- Tree & buffer modules + , Proven.SafeTree + , Proven.SafeBuffer + -- Policy & audit + , Proven.SafePolicy + , Proven.SafeProvenance + -- Infrastructure verification (trust, attestation, cycle detection) + , Proven.SafeAttestation + , Proven.SafeCycleDetect + , Proven.SafeTrust + -- State machines + , Proven.SafeStateMachine + -- Format safety + , Proven.SafeMarkdown + , Proven.SafeLog + , Proven.SafeCron + -- Numeric safety + , Proven.SafeFloat + , Proven.SafeDecimal + , Proven.SafeRational + , Proven.SafeComplex + -- Domain values + , Proven.SafeColor + , Proven.SafeVersion + , Proven.SafeChecksum + , Proven.SafeGeo + -- , Proven.SafeAngle -- MOVED to not-proven/ + -- Mathematics & physics + , Proven.SafeProbability + , Proven.SafeUnit + , Proven.SafeFiniteField + -- Data structures + , Proven.SafeQueue + , Proven.SafeBloom + , Proven.SafeLRU + , Proven.SafeSet + , Proven.SafeHeap + , Proven.SafeMatrix + , Proven.SafeBitset + , Proven.SafeInterval + , Proven.SafeUnionFind + -- Distributed systems + , Proven.SafeRateLimiter + , Proven.SafeCircuitBreaker + , Proven.SafeRetry + , Proven.SafeMonotonic + -- Advanced + , Proven.SafeGraph + , Proven.SafeResource + , Proven.SafeTransaction + , Proven.SafeCapability + , Proven.SafeConsensus + , Proven.SafeOrdering + , Proven.SafeSchema + , Proven.SafeTensor + -- Data formats + , Proven.SafeCSV + , Proven.SafeCSV.Proofs + , Proven.SafeDigest + , Proven.SafeDigest.Proofs + , Proven.SafeRegistry + -- Concurrency primitives + , Proven.SafeSemaphore + , Proven.SafeCalculator + -- New v1.1.0 modules + , Proven.SafeGit + , Proven.SafeDocker + , Proven.SafeDNS + , Proven.SafeSSH + , Proven.SafeI18n + , Proven.SafeTemplate + , Proven.SafeArchive + , Proven.SafeCert + , Proven.SafeOAuth + , Proven.SafeWebhook + , Proven.SafeBibTeX + , Proven.SafeMCP + , Proven.SafeML + -- Web security modules (v1.2.0) + , Proven.SafeHTTP + , Proven.SafePromptInjection + , Proven.SafeCORS + , Proven.SafeCORS.Proofs + , Proven.SafeAPIKey + , Proven.SafeAPIKey.Proofs + , Proven.SafeWebSocket + -- Security hardening modules (v1.2.0) + , Proven.SafeCSRF + , Proven.SafeCSP + , Proven.SafeRedirect + , Proven.SafeSRI + , Proven.SafeOTP + -- Security & identity (v1.3.0) + , Proven.SafeSSRF + , Proven.SafeRBAC + , Proven.SafeRateWindow + , Proven.SafeJWK + , Proven.SafeHSTS + , Proven.SafeSameSite + , Proven.SafeWebAuthn + , Proven.SafeHomoglyph + , Proven.SafeSupplyChain + -- Data integrity (v1.3.0) + , Proven.SafeSemVer + , Proven.SafeULID + , Proven.SafeMediaType + , Proven.SafeCBOR + -- Crypto parameter validation (v1.3.0) + , Proven.SafeHKDF + , Proven.SafePBKDF2 + , Proven.SafeSecretShare + -- Framework convenience re-export modules + , Proven.Network + , Proven.Crypto + , Proven.Hardware + , Proven.Data + , Proven.Collections + , Proven.System + , Proven.Safety + -- Hardware backend modules (GPU, VPU, TPU, crypto, FPGA, NPU, DSP, ISA) + , Proven.SafeGPU + , Proven.SafeVPU + , Proven.SafeTPU + , Proven.SafeCryptoAccel + , Proven.SafeFPGA + , Proven.SafeNPU + , Proven.SafeDSP + , Proven.SafeISA + -- Shell-specific modules (for valence-shell integration) + , Proven.SafeShell + , Proven.SafePipe + , Proven.SafeProcess + , Proven.SafeSignal + , Proven.SafeTerminal + -- FFI exports (connect Idris2 to Zig/C ABI) + , Proven.FFI + , Proven.FFI.SafePath + , Proven.FFI.SafeJson + , Proven.FFI.SafeMath + , Proven.FFI.SafeString + , Proven.FFI.SafeEmail + , Proven.FFI.SafeDateTime + , Proven.FFI.SafeHtml + , Proven.FFI.SafeCookie + , Proven.FFI.SafeHeader + , Proven.FFI.SafeContentType + , Proven.FFI.SafeJWT + , Proven.FFI.SafeSQL + , Proven.FFI.SafeCommand + , Proven.FFI.SafeCrypto + , Proven.FFI.SafePassword + , Proven.FFI.SafeRegex + , Proven.FFI.SafeXML + , Proven.FFI.SafeYAML + , Proven.FFI.SafeTOML + , Proven.FFI.SafeNetwork + , Proven.FFI.SafeUrl + , Proven.FFI.SafeEnv + , Proven.FFI.SafeArgs + , Proven.FFI.SafeFile + , Proven.FFI.SafeUUID + , Proven.FFI.SafeCurrency + , Proven.FFI.SafeHex + , Proven.FFI.SafeTree + , Proven.FFI.SafeBuffer + , Proven.FFI.SafePolicy + , Proven.FFI.SafeProvenance + , Proven.FFI.SafeStateMachine + , Proven.FFI.SafeMarkdown + , Proven.FFI.SafeLog + , Proven.FFI.SafeCron + , Proven.FFI.SafeFloat + , Proven.FFI.SafeColor + , Proven.FFI.SafeVersion + , Proven.FFI.SafeChecksum + , Proven.FFI.SafeGeo + , Proven.FFI.SafeAngle + , Proven.FFI.SafeProbability + , Proven.FFI.SafeUnit + , Proven.FFI.SafeQueue + , Proven.FFI.SafeBloom + , Proven.FFI.SafeLRU + , Proven.FFI.SafeRateLimiter + , Proven.FFI.SafeCircuitBreaker + , Proven.FFI.SafeRetry + , Proven.FFI.SafeMonotonic + , Proven.FFI.SafeGraph + , Proven.FFI.SafeResource + , Proven.FFI.SafeTransaction + , Proven.FFI.SafeCapability + , Proven.FFI.SafeConsensus + , Proven.FFI.SafeFiniteField + , Proven.FFI.SafeOrdering + , Proven.FFI.SafeSchema + , Proven.FFI.SafeTensor + , Proven.FFI.SafeGit + , Proven.FFI.SafeDocker + , Proven.FFI.SafeDNS + , Proven.FFI.SafeSSH + , Proven.FFI.SafeI18n + , Proven.FFI.SafeTemplate + , Proven.FFI.SafeArchive + , Proven.FFI.SafeCert + , Proven.FFI.SafeOAuth + , Proven.FFI.SafeWebhook + , Proven.FFI.SafeBibTeX + , Proven.FFI.SafeMCP + , Proven.FFI.SafeML + , Proven.FFI.SafePhone + -- ECHIDNA neurosymbolic theorem prover validation + -- NOTE: ECHIDNA modules temporarily excluded - need redesign + -- (proof keyword, ProofValid forward ref, All ambiguity, verify resolution) + -- , Proven.ECHIDNA.ProofTerm + -- , Proven.ECHIDNA.Validator + -- , Proven.ECHIDNA.Soundness + +depends = base >= 0.6.0 + , contrib + , network + +opts = "--total" diff --git a/idris-dependents/tests/oracle.rs b/idris-dependents/tests/oracle.rs new file mode 100644 index 00000000..f83a109a --- /dev/null +++ b/idris-dependents/tests/oracle.rs @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Differential test against Idris2 itself: for the real fixtures and for +//! generated files, `depends()` must agree with `idris2 --dump-ipkg-json`. +//! +//! Runs only when `IDRIS2` names an Idris2 binary new enough to have +//! `--dump-ipkg-json` (Idris2 `main`, not 0.7.0); otherwise it is skipped. +//! `ORACLE_CASES` sets how many generated files to check (default 300). + +mod common; + +use std::path::Path; +use std::process::Command; + +use idris_dependents::ipkg::{depends, package_name}; +use proptest::strategy::{Strategy, ValueTree}; +use proptest::test_runner::TestRunner; + +/// (package name, depends) as Idris2 reads the file, or Idris2's error. +fn idris_reads(idris2: &str, file: &Path) -> Result<(String, Vec), String> { + let out = Command::new(idris2) + .arg("--dump-ipkg-json") + .arg(file) + .output() + .map_err(|e| e.to_string())?; + let stdout = String::from_utf8_lossy(&out.stdout); + if !out.status.success() { + return Err(format!("{stdout}{}", String::from_utf8_lossy(&out.stderr))); + } + let v: serde_json::Value = + serde_json::from_str(stdout.trim()).map_err(|e| format!("{e}: {stdout}"))?; + let name = v["name"].as_str().unwrap_or_default().to_string(); + let deps = v["depends"] + .as_array() + .map(|a| { + a.iter() + .filter_map(|d| d.as_object()?.keys().next().cloned()) + .collect() + }) + .unwrap_or_default(); + Ok((name, deps)) +} + +#[test] +fn agrees_with_idris2() { + let Ok(idris2) = std::env::var("IDRIS2") else { + eprintln!("skipped: set IDRIS2 to an idris2 binary with --dump-ipkg-json"); + return; + }; + let n: usize = std::env::var("ORACLE_CASES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(300); + let dir = std::env::temp_dir().join(format!("idris-dependents-oracle-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + + let fixtures = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"); + let mut cases: Vec<(String, String)> = std::fs::read_dir(&fixtures) + .unwrap() + .filter_map(|e| e.ok()?.path().to_str().map(String::from)) + .filter(|p| p.ends_with(".ipkg")) + .map(|p| (p.clone(), std::fs::read_to_string(&p).unwrap())) + .collect(); + let mut runner = TestRunner::deterministic(); + for i in 0..n { + cases.push(( + format!("generated #{i}"), + common::ipkg().new_tree(&mut runner).unwrap().current().text, + )); + } + + let mut mismatches = Vec::new(); + for (i, (label, text)) in cases.iter().enumerate() { + let file = dir.join(format!("case{i}.ipkg")); + std::fs::write(&file, text).unwrap(); + match idris_reads(&idris2, &file) { + Ok((name, deps)) => { + let ours = (package_name(text).unwrap_or_default(), depends(text)); + if ours != (name.clone(), deps.clone()) { + mismatches.push(format!( + "{label}: idris2={name} {deps:?} ours={ours:?}\n{text}" + )); + } + } + Err(e) => mismatches.push(format!("{label}: idris2 rejected the file: {e}\n{text}")), + } + } + let _ = std::fs::remove_dir_all(&dir); + assert!( + mismatches.is_empty(), + "{} of {} cases disagree with Idris2; first few:\n\n{}", + mismatches.len(), + cases.len(), + mismatches + .iter() + .take(5) + .cloned() + .collect::>() + .join("\n---\n") + ); + eprintln!("idris2 agrees on all {} cases", cases.len()); +} diff --git a/idris-dependents/tests/roundtrip_props.rs b/idris-dependents/tests/roundtrip_props.rs new file mode 100644 index 00000000..5c597e62 --- /dev/null +++ b/idris-dependents/tests/roundtrip_props.rs @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Property: printing a random well-formed `.ipkg` and parsing it back +//! recovers exactly the package name and dependency list that were printed, +//! whatever comments, line endings and comma styles surround them. + +mod common; + +use idris_dependents::ipkg::{depends, package_name}; +use proptest::prelude::*; + +proptest! { + #![proptest_config(ProptestConfig::with_cases(2000))] + + #[test] + fn depends_round_trip(p in common::ipkg()) { + prop_assert_eq!(depends(&p.text), p.deps, "file:\n{}", p.text); + prop_assert_eq!(package_name(&p.text), Some(p.name), "file:\n{}", p.text); + } + + #[test] + fn lexer_is_total(s in "\\PC{0,200}") { + // Arbitrary input never panics, whatever it contains. + let _ = depends(&s); + let _ = package_name(&s); + } +} diff --git a/idris-dependents/tests/rules_tests.rs b/idris-dependents/tests/rules_tests.rs new file mode 100644 index 00000000..3ed461cc --- /dev/null +++ b/idris-dependents/tests/rules_tests.rs @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Parser and classification rules. Ported from the original lib.test.js, +//! plus regressions for bugs found in that implementation (P1–P6). + +use idris_dependents::catalogue::{classify, is_distinctive, search_stems, Confidence}; +use idris_dependents::ipkg::{depends, package_name}; +use idris_dependents::refs::{gitmodules_refs, pack_toml_refs, points_at}; + +fn fx(name: &str) -> String { + std::fs::read_to_string(format!( + "{}/tests/fixtures/{name}", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap() +} +fn own() -> Vec { + vec!["hyperpolymath".into(), "metadatastician".into()] +} +fn s(v: &[&str]) -> Vec { + v.iter().map(|x| x.to_string()).collect() +} + +// --- parsers on real estate files ----------------------------------------------- + +#[test] +fn real_ochrance_ipkg_name_and_multiline_depends() { + let t = fx("ochrance.ipkg"); + assert_eq!(package_name(&t).as_deref(), Some("ochrance")); + assert_eq!(depends(&t), s(&["base", "contrib", "network"])); +} + +#[test] +fn real_proven_ipkg_version_bound_stripped() { + let t = fx("proven.ipkg"); + assert_eq!(package_name(&t).as_deref(), Some("proven")); + assert_eq!(depends(&t), s(&["base", "contrib", "network"])); +} + +#[test] +fn real_ordering_laws_ipkg_two_of_your_own_packages() { + let t = fx("ordering-laws.ipkg"); + assert_eq!(package_name(&t).as_deref(), Some("proven-ordering-laws")); + assert_eq!(depends(&t), s(&["proven-tests", "proven-safeordering"])); +} + +#[test] +fn real_pack_toml_local_path_entry_is_not_a_github_ref() { + assert_eq!(pack_toml_refs(&fx("pack.toml"), &own()).unwrap(), vec![]); +} + +// --- edge cases ----------------------------------------------------------------- + +#[test] +fn depends_comments_leading_commas_bounds_trailing_fields() { + let t = "package foo +-- depends = fake +{- depends = alsofake -} +depends = base >= 0.6.0 && < 0.8 -- inline comment + , ochrance + , proven-dns +modules = Foo +opts = \"--codegen chez\""; + assert_eq!(depends(t), s(&["base", "ochrance", "proven-dns"])); +} + +#[test] +fn no_depends_field() { + assert!(depends("package x\nmodules = A").is_empty()); +} + +#[test] +fn depends_field_as_the_last_line() { + assert_eq!(depends("package x\ndepends = trope"), s(&["trope"])); +} + +#[test] +fn commented_out_entry_inside_multiline_depends_does_not_leak() { + assert_eq!( + depends("package foo\ndepends = base\n -- , oldpkg\n , ochrance"), + s(&["base", "ochrance"]) + ); +} + +#[test] +fn whole_depends_line_commented_out() { + assert!(depends("package proven-dns\n-- depends = proven\nmodules = DNS").is_empty()); +} + +#[test] +fn multiline_block_comment_inside_depends_list_is_removed() { + let t = "package foo\ndepends = base,\n {- proven-old,\n legacy -}\n ochrance"; + assert_eq!(depends(t), s(&["base", "ochrance"])); +} + +#[test] +fn pack_toml_github_url_to_you_counts_others_do_not_quoted_keys_ok() { + let t = r#"[custom.all.proven] +type = "github" +url = "https://github.com/hyperpolymath/proven" +commit = "abc" +[custom.all."lsp-lib"] +type = "github" +url = "https://github.com/idris-community/LSP-lib" +[custom.all.tropecheck] +url = "git@github.com:Hyperpolymath/trope-checker.git" # ssh form, mixed case"#; + let names: Vec = pack_toml_refs(t, &own()) + .unwrap() + .into_iter() + .map(|r| r.name) + .collect(); + assert_eq!(names, s(&["proven", "tropecheck"])); +} + +#[test] +fn gitmodules_refs_basic() { + let t = "[submodule \"x\"]\n\tpath = vendor/proven\n\turl = https://github.com/hyperpolymath/proven.git\n[submodule \"y\"]\n\turl = https://github.com/someone/else"; + assert_eq!( + gitmodules_refs(t, &own()), + s(&["https://github.com/hyperpolymath/proven.git"]) + ); +} + +#[test] +fn owner_match_is_exact_not_a_prefix() { + assert!(gitmodules_refs("url = https://github.com/hyperpolymathx/proven", &own()).is_empty()); +} + +#[test] +fn distinctive_filter_drops_internal_test_bench_packages() { + for n in [ + "abi", + "benchmarks", + "coreonly", + "test-ffi", + "proven-tests", + "proven-spec-suite", + "proven-tests-bench", + ] { + assert!(!is_distinctive(n), "{n}"); + } + for n in [ + "proven", + "proven-dns", + "ochrance", + "typedwasm", + "phi-lam", + "nimiser-abi", + ] { + assert!(is_distinctive(n), "{n}"); + } +} + +#[test] +fn stems_collapse_families_keep_short_stem_names_whole() { + assert_eq!( + search_stems(&s(&[ + "proven", + "proven-dns", + "proven-ffi", + "phi-lam", + "ochrance", + "test-ffi" + ])), + s(&["ochrance", "phi-lam", "proven"]) + ); +} + +#[test] +fn classification_ladder() { + assert_eq!(classify(&[], 1, 0), Some(Confidence::High)); + assert_eq!( + classify(&s(&["proven-dns"]), 0, 0), + Some(Confidence::Medium) + ); + assert_eq!(classify(&s(&["proven"]), 0, 0), Some(Confidence::Low)); + assert_eq!(classify(&[], 0, 0), None); +} + +// --- regressions for bugs in the JS implementation ---------------------------- + +#[test] +fn p1_crlf_line_endings() { + assert_eq!( + depends("package x\r\ndepends = base, ochrance\r\nmodules = A\r\n"), + s(&["base", "ochrance"]) + ); + assert_eq!(package_name("package x\r\n").as_deref(), Some("x")); +} + +#[test] +fn p2_eq_bound_on_continuation_line() { + let t = "package x\ndepends = base,\n ochrance == 0.1.0,\n proven-dns\nmodules = A"; + assert_eq!(depends(t), s(&["base", "ochrance", "proven-dns"])); +} + +#[test] +fn p3_block_opener_inside_line_comment_is_inert() { + assert_eq!( + depends("package x\n-- see {- note\ndepends = ochrance\n-- end -}\nmodules = A"), + s(&["ochrance"]) + ); +} + +#[test] +fn p4_nested_block_comments() { + assert_eq!( + depends("package x\n{- outer {- inner -} depends = fake -}\ndepends = ochrance"), + s(&["ochrance"]) + ); +} + +#[test] +fn p5_hash_inside_quoted_pack_toml_url() { + assert_eq!( + pack_toml_refs( + "[custom.all.p]\nurl = \"https://github.com/hyperpolymath/p#readme\"", + &own() + ) + .unwrap() + .len(), + 1 + ); +} + +#[test] +fn p6_single_quoted_pack_toml_url() { + assert_eq!( + pack_toml_refs( + "[custom.all.p]\nurl = 'https://github.com/hyperpolymath/p'", + &own() + ) + .unwrap() + .len(), + 1 + ); +} + +// --- Idris2 grammar details (Parser/Lexer/Common.idr on Idris2 main) ---------- + +#[test] +fn dash_dash_inside_block_comment_hides_close_on_same_line() { + // `--` inside a block comment is a line comment, so this `-}` does not close it. + let t = "package x\n{- a -- b -}\ndepends = fake\n-}\ndepends = ochrance"; + assert_eq!(depends(t), s(&["ochrance"])); +} + +#[test] +fn close_delimiter_inside_string_inside_block_comment_is_inert() { + let t = "package x\n{- \"-}\" depends = fake -}\ndepends = ochrance"; + assert_eq!(depends(t), s(&["ochrance"])); +} + +#[test] +fn dashes_in_strings_are_not_comments() { + let t = "package x\nopts = \"--total -- depends = fake\"\ndepends = ochrance"; + assert_eq!(depends(t), s(&["ochrance"])); +} + +#[test] +fn trailing_comma_does_not_swallow_next_field_key() { + assert_eq!( + depends("package x\ndepends = base,\nmodules = A"), + s(&["base"]) + ); +} + +#[test] +fn several_depends_fields_and_duplicates() { + assert_eq!( + depends("package x\ndepends = a, b\nmodules = M\ndepends = b, c"), + s(&["a", "b", "c"]) + ); +} + +#[test] +fn module_names_are_not_package_names() { + assert_eq!( + depends("package x\ndepends = Proven.SafeMath, ochrance"), + Vec::::new() + ); +} + +#[test] +fn package_needs_a_name_not_a_field() { + assert_eq!(package_name("package = 3"), None); +} + +#[test] +fn host_must_be_github_exactly() { + let o = own(); + assert!(points_at("https://github.com/hyperpolymath/proven", &o)); + assert!(points_at("ssh://git@github.com/hyperpolymath/proven", &o)); + assert!(points_at("git@github.com:metadatastician/x.git", &o)); + assert!(points_at("https://www.github.com/hyperpolymath/proven", &o)); + assert!(!points_at("https://notgithub.com/hyperpolymath/proven", &o)); + assert!(!points_at("https://github.com/hyperpolymath", &o)); +} + +#[test] +fn gitmodules_quoted_and_crlf() { + let t = "[submodule \"x\"]\r\n\turl = \"https://github.com/hyperpolymath/proven\"\r\n"; + assert_eq!( + gitmodules_refs(t, &own()), + s(&["https://github.com/hyperpolymath/proven"]) + ); +} diff --git a/idris-dependents/tests/scan_tests.rs b/idris-dependents/tests/scan_tests.rs new file mode 100644 index 00000000..fe195f11 --- /dev/null +++ b/idris-dependents/tests/scan_tests.rs @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: MPL-2.0 +//! End-to-end scan against a mocked GitHub (ported from lib.test.js). + +use std::collections::BTreeMap; +use std::time::Duration; + +use anyhow::Result; +use idris_dependents::catalogue::Confidence; +use idris_dependents::scan::{run, Io, Options, Response}; +use serde_json::json; + +struct Mock { + searches: BTreeMap<&'static str, Vec>, + contents: BTreeMap<&'static str, &'static str>, + limited_once: bool, + logs: Vec, + gets: usize, +} + +fn item(full: &str, path: &str, url: &str) -> serde_json::Value { + json!({ + "url": url, + "html_url": format!("https://github.com/{full}/blob/main/{path}"), + "repository": { "full_name": full, "owner": { "login": full.split('/').next().unwrap() } } + }) +} + +fn decode(q: &str) -> String { + let mut out = Vec::new(); + let b = q.as_bytes(); + let mut i = 0; + while i < b.len() { + if b[i] == b'%' { + out.push(u8::from_str_radix(&q[i + 1..i + 3], 16).unwrap()); + i += 3; + } else { + out.push(b[i]); + i += 1; + } + } + String::from_utf8(out).unwrap() +} + +impl Io for Mock { + fn get(&mut self, url: &str, raw: bool) -> Result { + self.gets += 1; + let ok = |body: String| Response { + status: 200, + headers: BTreeMap::new(), + body, + }; + if url.contains("/search/code") { + assert!(!raw); + if !self.limited_once { + self.limited_once = true; + let headers = BTreeMap::from([("retry-after".to_string(), "1".to_string())]); + return Ok(Response { + status: 429, + headers, + body: String::new(), + }); + } + let q = decode(url.split("q=").nth(1).unwrap().split('&').next().unwrap()); + let items = self.searches.get(q.as_str()).cloned().unwrap_or_default(); + return Ok(ok( + json!({ "total_count": items.len(), "items": items }).to_string() + )); + } + assert!(raw); + Ok(ok(self.contents[url].to_string())) + } + fn sleep(&mut self, _: Duration) {} + fn now_secs(&self) -> u64 { + 0 + } + fn log(&mut self, msg: &str) { + self.logs.push(msg.to_string()); + } +} + +#[test] +fn run_filters_self_dedups_files_classifies_survives_rate_limit() { + let mut io = Mock { + searches: BTreeMap::from([ + ( + "proven extension:ipkg", + vec![ + item("alice/app", "app.ipkg", "c/alice"), + item("alice/app", "app.ipkg", "c/alice"), // duplicate file + item("bob/b", "b.ipkg", "c/bob"), + item("hyperpolymath/x", "x.ipkg", "c/self"), + item("dave/d", "d.ipkg", "c/dave"), + item("erin/e", "e.ipkg", "c/erin-crlf"), + ], + ), + ( + "hyperpolymath filename:pack.toml", + vec![item("carol/c", "pack.toml", "c/carol-pack")], + ), + ]), + contents: BTreeMap::from([ + ("c/alice", "package app\ndepends = base, proven-dns"), + ("c/bob", "package b\ndepends = proven"), // ambiguous -> LOW + ( + "c/carol-pack", + "[custom.all.ochrance]\nurl = \"https://github.com/hyperpolymath/ochrance\"", + ), + ("c/self", "package mine\ndepends = proven-dns"), // own account -> skipped + ("c/dave", "package d\ndepends = base, contrib"), // matched search, no real dep + ( + "c/erin-crlf", + "package e\r\ndepends = base,\r\n ochrance == 0.1.0\r\n", + ), // P1+P2 in the wild + ]), + limited_once: false, + logs: vec![], + gets: 0, + }; + let opts = Options { + owners: vec!["hyperpolymath".into()], + self_accounts: vec!["hyperpolymath".into()], + packages: Some(vec![ + "proven".into(), + "proven-dns".into(), + "ochrance".into(), + "proven-tests".into(), + ]), + }; + let res = run(&opts, &mut io).unwrap(); + let got: Vec<(&str, Confidence)> = res + .rows + .iter() + .map(|r| (r.repo.as_str(), r.confidence)) + .collect(); + assert_eq!( + got, + vec![ + ("carol/c", Confidence::High), + ("alice/app", Confidence::Medium), + ("erin/e", Confidence::Medium), + ("bob/b", Confidence::Low), + ] + ); + assert_eq!( + res.rows + .iter() + .find(|r| r.repo == "alice/app") + .unwrap() + .files + .len(), + 1 + ); + assert_eq!(res.catalogue_size, 3); + assert!(io.logs.iter().any(|l| l.starts_with("rate-limited"))); + assert!(res.incomplete_queries.is_empty()); +} + +#[test] +fn truncated_search_is_reported_not_silent() { + struct Big; + impl Io for Big { + fn get(&mut self, _: &str, _: bool) -> Result { + let body = json!({ "total_count": 5000, "incomplete_results": false, "items": [] }) + .to_string(); + Ok(Response { + status: 200, + headers: BTreeMap::new(), + body, + }) + } + fn sleep(&mut self, _: Duration) {} + fn now_secs(&self) -> u64 { + 0 + } + fn log(&mut self, _: &str) {} + } + let opts = Options { + owners: vec!["o".into()], + self_accounts: vec!["o".into()], + packages: Some(vec!["ochrance".into()]), + }; + let res = run(&opts, &mut Big).unwrap(); + assert_eq!( + res.incomplete_queries, + vec![ + "ochrance extension:ipkg", + "o filename:pack.toml", + "o filename:.gitmodules" + ] + ); +} + +#[test] +fn non_rate_limit_errors_fail_loudly() { + struct Forbidden; + impl Io for Forbidden { + fn get(&mut self, _: &str, _: bool) -> Result { + Ok(Response { + status: 401, + headers: BTreeMap::new(), + body: String::new(), + }) + } + fn sleep(&mut self, _: Duration) {} + fn now_secs(&self) -> u64 { + 0 + } + fn log(&mut self, _: &str) {} + } + let opts = Options { + owners: vec!["o".into()], + self_accounts: vec![], + packages: Some(vec!["ochrance".into()]), + }; + let err = run(&opts, &mut Forbidden).unwrap_err().to_string(); + assert!(err.contains("HTTP 401"), "{err}"); +} From 5d234603853c2c85f4a4410c7f7ba87e8f74c2a7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 08:48:15 +0000 Subject: [PATCH 2/6] test(idris-dependents): make the Idris2 oracle runnable `idris2 --dump-ipkg-json` checks that each listed module exists and may print deprecation warnings before the JSON. Give every case its own directory with stub modules under its sourcedir, and read the JSON line rather than all of stdout. Result with Idris2 0.8.0-d87ae51c3 (main): idris2 agrees on all 1003 cases (3 real fixtures + 1000 generated files). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01J4QvDfb1gn2SD218jMg9jk --- idris-dependents/tests/oracle.rs | 54 +++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/idris-dependents/tests/oracle.rs b/idris-dependents/tests/oracle.rs index f83a109a..b8ebd93a 100644 --- a/idris-dependents/tests/oracle.rs +++ b/idris-dependents/tests/oracle.rs @@ -11,23 +11,64 @@ mod common; use std::path::Path; use std::process::Command; -use idris_dependents::ipkg::{depends, package_name}; +use idris_dependents::ipkg::{depends, lex, package_name, Tok}; use proptest::strategy::{Strategy, ValueTree}; use proptest::test_runner::TestRunner; +/// Idris2 checks that every `modules` / `main` entry exists under `sourcedir`, +/// so give each case an empty source file for each module it names. +fn stub_modules(dir: &Path, text: &str) { + let (toks, _) = lex(text); + let mut sourcedir = ".".to_string(); + let mut modules = Vec::new(); + for i in 0..toks.len() { + let key = match &toks[i] { + Tok::Ident { + text, + dotted: false, + } if toks.get(i + 1) == Some(&Tok::Equals) => text.as_str(), + _ => continue, + }; + match (key, toks.get(i + 2)) { + ("sourcedir", Some(Tok::Str(s))) => sourcedir = s.clone(), + ("modules" | "main", _) => { + let mut j = i + 2; + while let Some(Tok::Ident { text, .. }) = toks.get(j) { + modules.push(text.clone()); + if toks.get(j + 1) != Some(&Tok::Sep) { + break; + } + j += 2; + } + } + _ => {} + } + } + for m in modules { + let file = dir.join(&sourcedir).join(m.replace('.', "/") + ".idr"); + std::fs::create_dir_all(file.parent().unwrap()).unwrap(); + std::fs::write(file, format!("module {m}\n")).unwrap(); + } +} + /// (package name, depends) as Idris2 reads the file, or Idris2's error. fn idris_reads(idris2: &str, file: &Path) -> Result<(String, Vec), String> { let out = Command::new(idris2) .arg("--dump-ipkg-json") - .arg(file) + .arg(file.file_name().unwrap()) + .current_dir(file.parent().unwrap()) .output() .map_err(|e| e.to_string())?; let stdout = String::from_utf8_lossy(&out.stdout); if !out.status.success() { return Err(format!("{stdout}{}", String::from_utf8_lossy(&out.stderr))); } - let v: serde_json::Value = - serde_json::from_str(stdout.trim()).map_err(|e| format!("{e}: {stdout}"))?; + // Warnings (e.g. a deprecated `version = "x.y"` string) precede the JSON. + let json = stdout + .lines() + .find(|l| l.starts_with('{')) + .unwrap_or_default(); + let v: serde_json::Value = serde_json::from_str(json).map_err(|e| format!("{e}: {stdout}"))?; let name = v["name"].as_str().unwrap_or_default().to_string(); let deps = v["depends"] .as_array() @@ -70,7 +111,10 @@ fn agrees_with_idris2() { let mut mismatches = Vec::new(); for (i, (label, text)) in cases.iter().enumerate() { - let file = dir.join(format!("case{i}.ipkg")); + let case_dir = dir.join(format!("case{i}")); + std::fs::create_dir_all(&case_dir).unwrap(); + stub_modules(&case_dir, text); + let file = case_dir.join("pkg.ipkg"); std::fs::write(&file, text).unwrap(); match idris_reads(&idris2, &file) { Ok((name, deps)) => { From 5329daa1aba2f89f752567f62d993aafcf8659d2 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:34:03 +0000 Subject: [PATCH 3/6] Update idris-dependents/src/scan.rs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: Jonathan D.A. Jewell <6759885+hyperpolymath@users.noreply.github.com> --- idris-dependents/src/scan.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/idris-dependents/src/scan.rs b/idris-dependents/src/scan.rs index 003c187e..5963d52e 100644 --- a/idris-dependents/src/scan.rs +++ b/idris-dependents/src/scan.rs @@ -235,13 +235,13 @@ pub fn run(opts: &Options, io: &mut I) -> Result { acc.owner = it.owner.clone(); let found = match kind { Kind::Ipkg => { - let before = acc.deps.len(); - acc.deps.extend( - ipkg::depends(&text) - .into_iter() - .filter(|d| distinctive.contains(d.as_str())), - ); - acc.deps.len() > before + let matched: Vec = ipkg::depends(&text) + .into_iter() + .filter(|d| distinctive.contains(d.as_str())) + .collect(); + let found = !matched.is_empty(); + acc.deps.extend(matched); + found } Kind::Pack => match refs::pack_toml_refs(&text, &opts.owners) { Ok(r) => { From 57ec6093f83f118f3aac68757c4fe2f47778ecf4 Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:36:56 +0000 Subject: [PATCH 4/6] docs(idris-dependents): document parser, reference matching, and scanner behavior --- idris-dependents/src/catalogue.rs | 9 +++++++- idris-dependents/src/ipkg.rs | 15 +++++++++++++- idris-dependents/src/main.rs | 13 ++++++++++++ idris-dependents/src/refs.rs | 15 ++++++++++---- idris-dependents/src/scan.rs | 34 +++++++++++++++++++++++++++++++ 5 files changed, 80 insertions(+), 6 deletions(-) diff --git a/idris-dependents/src/catalogue.rs b/idris-dependents/src/catalogue.rs index dbfb749c..b9b1113e 100644 --- a/idris-dependents/src/catalogue.rs +++ b/idris-dependents/src/catalogue.rs @@ -4,7 +4,8 @@ use serde::Serialize; -/// Internal test/bench/ABI-shim packages nobody else would depend on by name. +/// Whether a name passes the case-insensitive filter for internal test, +/// benchmark, and ABI-shim package names. pub fn is_distinctive(name: &str) -> bool { let n = name.to_ascii_lowercase(); let internal = matches!( @@ -31,6 +32,7 @@ pub const AMBIGUOUS: &[&str] = &["proven", "trope"]; /// One search term per package family: `proven-ffi`, `proven-dns`… collapse to /// `proven`. Stems shorter than five characters are too noisy to search, so /// those names are kept whole. +/// Internal package names are excluded; results are sorted and deduplicated. pub fn search_stems(names: &[String]) -> Vec { let mut stems: Vec = names .iter() @@ -60,6 +62,11 @@ pub enum Confidence { Low, } +/// Classify already-matched dependencies and counts of owner references. +/// +/// Any pack or submodule reference yields high confidence. Otherwise, a name +/// outside [`AMBIGUOUS`] yields medium confidence, only ambiguous names yield +/// low confidence, and no evidence yields `None`. Name comparison is case-sensitive. pub fn classify(deps: &[String], pack_refs: usize, sub_refs: usize) -> Option { if pack_refs > 0 || sub_refs > 0 { Some(Confidence::High) diff --git a/idris-dependents/src/ipkg.rs b/idris-dependents/src/ipkg.rs index 5e18c4b8..2c16a310 100644 --- a/idris-dependents/src/ipkg.rs +++ b/idris-dependents/src/ipkg.rs @@ -32,16 +32,23 @@ pub enum Tok { Int(String), } +/// Whether a character can start an identifier: ASCII letters, `_`, or any +/// character above U+00A0. fn is_ident_start(c: char) -> bool { c == '_' || c.is_ascii_alphabetic() || c > '\u{a0}' } +/// Whether a character can follow an identifier's first character, including +/// ASCII digits, hyphens, and apostrophes. fn is_ident_trailing(c: char) -> bool { c == '-' || c == '\'' || c == '_' || c.is_ascii_alphanumeric() || c > '\u{a0}' } -/// Tokenise `src`. Returns the tokens and the number of characters skipped +/// Tokenize `src`. Returns the tokens and the number of characters skipped /// because no Idris2 token rule matches them. +/// Comments and whitespace are omitted. An unterminated block comment consumes +/// the remaining input; an unterminated string's opening quote is counted as +/// skipped, and tokenization continues after it. String contents retain escapes. pub fn lex(src: &str) -> (Vec, usize) { let s: Vec = src.chars().collect(); let at = |j: usize| s.get(j).copied(); @@ -133,6 +140,8 @@ pub fn lex(src: &str) -> (Vec, usize) { } /// Index just past the closing quote of the string literal opening at `i`. +/// Backslash escapes skip the next character; returns `None` if no closing +/// quote is found. Indices count characters, not UTF-8 bytes. fn string_end(s: &[char], i: usize) -> Option { let mut j = i + 1; while j < s.len() { @@ -149,6 +158,8 @@ fn string_end(s: &[char], i: usize) -> Option { /// `toEndComment` / `singleBrace` / `singleDash` / `doubleDash` automaton: /// comments nest, string and character literals inside them are skipped, /// and `--` inside a comment hides the rest of its line (including any `-}`). +/// Returns the character index after the closing delimiter, or `s.len()` if +/// the comment is unterminated. fn block_comment(s: &[char], i: usize) -> usize { let at = |j: usize| s.get(j).copied(); let mut j = i + 2; @@ -213,6 +224,8 @@ fn block_comment(s: &[char], i: usize) -> usize { } } +/// Borrow the text of an undotted identifier, or return `None` for any other +/// token or missing token. fn ident(t: Option<&Tok>) -> Option<&str> { match t { Some(Tok::Ident { diff --git a/idris-dependents/src/main.rs b/idris-dependents/src/main.rs index f28e3cd8..400a916a 100644 --- a/idris-dependents/src/main.rs +++ b/idris-dependents/src/main.rs @@ -65,6 +65,8 @@ struct Live { } impl Io for Live { + /// Send an authenticated GET requesting raw contents or JSON. Preserve the + /// HTTP status for the scanner; request and response-body errors propagate. fn get(&mut self, url: &str, raw: bool) -> Result { let accept = if raw { "application/vnd.github.raw" @@ -97,20 +99,26 @@ impl Io for Live { body: res.text()?, }) } + /// Block the current thread for the requested duration. fn sleep(&mut self, d: Duration) { std::thread::sleep(d); } + /// Return Unix time in whole seconds, falling back to zero for a clock + /// before the Unix epoch. fn now_secs(&self) -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0) } + /// Write a diagnostic message to standard error. fn log(&mut self, msg: &str) { eprintln!("{msg}"); } } +/// Print the scan summary, repository evidence, and incomplete queries to +/// standard output in the report's existing row order. fn print_report(r: &Report) { let owners: std::collections::BTreeSet<&str> = r.rows.iter().map(|x| x.owner.as_str()).collect(); @@ -152,6 +160,11 @@ fn print_report(r: &Report) { } } +/// Run an offline parser command or an authenticated GitHub scan, printing +/// results to standard output and optionally writing the scan report as JSON. +/// +/// Return errors for unreadable input, invalid TOML, missing scan credentials, +/// client construction, scanning, JSON serialization, or report-file writes. fn main() -> Result<()> { match Cli::parse().command { Command::ParseIpkg { file } => { diff --git a/idris-dependents/src/refs.rs b/idris-dependents/src/refs.rs index 7d00e14c..5e87859f 100644 --- a/idris-dependents/src/refs.rs +++ b/idris-dependents/src/refs.rs @@ -11,9 +11,12 @@ pub struct PackRef { pub url: String, } -/// True when `url` is a GitHub URL (https, ssh or scp form) under one of -/// `owners`. The owner must match exactly (case-insensitive), and `github.com` -/// must be the whole host, so `notgithub.com/owner/x` does not count. +/// Match a watched owner in a GitHub-style repository reference, including +/// HTTPS, SSH, and scp forms. Owner matching is ASCII case-insensitive and +/// requires a following `/`. +/// +/// This is a textual check, not full URL validation: `github.com` may follow +/// `/`, `@`, or `.` (or start the input), so subdomains and embedded paths can match. pub fn points_at(url: &str, owners: &[String]) -> bool { let url = url.to_ascii_lowercase(); let mut from = 0; @@ -39,6 +42,8 @@ pub fn points_at(url: &str, owners: &[String]) -> bool { /// `pack.toml` custom entries pointing at `owners`, in file order. /// Uses a real TOML parser, so quoting styles and `#` inside strings are /// handled the way pack itself handles them. +/// Missing custom tables or entries without matching string URLs are ignored. +/// Returns a TOML parse error if the document is invalid. pub fn pack_toml_refs(text: &str, owners: &[String]) -> Result, toml::de::Error> { let doc: toml::Table = text.parse()?; let mut refs = Vec::new(); @@ -60,7 +65,9 @@ pub fn pack_toml_refs(text: &str, owners: &[String]) -> Result, tom Ok(refs) } -/// Submodule URLs in `.gitmodules` (git-config syntax) pointing at `owners`. +/// Extract URLs matching `owners` from trimmed `url =` lines, in input order. +/// Double-quoted values end at the next quote; unquoted values end at `#` or +/// `;`. Sections and escapes are not interpreted, and duplicates are retained. pub fn gitmodules_refs(text: &str, owners: &[String]) -> Vec { text.lines() .filter_map(|line| { diff --git a/idris-dependents/src/scan.rs b/idris-dependents/src/scan.rs index 5963d52e..d23d5c37 100644 --- a/idris-dependents/src/scan.rs +++ b/idris-dependents/src/scan.rs @@ -26,9 +26,15 @@ pub struct Response { /// Side effects the scan needs. `raw` asks for file contents instead of JSON. pub trait Io { + /// Fetch `url`, requesting raw file contents when `raw` is true and JSON + /// otherwise. Return HTTP statuses for the scanner to handle; request or + /// body-reading errors propagate to the caller. fn get(&mut self, url: &str, raw: bool) -> Result; + /// Wait for the requested duration for search pacing or rate-limit retries. fn sleep(&mut self, d: Duration); + /// Current Unix time in seconds, used to interpret rate-limit reset headers. fn now_secs(&self) -> u64; + /// Emit a scan progress or diagnostic message. fn log(&mut self, msg: &str); } @@ -90,6 +96,12 @@ struct Scanner<'a, I: Io> { } impl Scanner<'_, I> { + /// Fetch a response body, allowing up to six attempts for HTTP 429 or + /// HTTP 403 with `retry-after` or `x-ratelimit-remaining: 0`. Wait according + /// to retry/reset headers, falling back to 60 seconds when neither is usable. + /// + /// Propagate I/O errors immediately. Other non-2xx statuses and exhausted + /// rate-limit retries return errors. fn get(&mut self, url: &str, raw: bool) -> Result { for _ in 0..=5 { let res = self.io.get(url, raw)?; @@ -116,6 +128,12 @@ impl Scanner<'_, I> { bail!("still rate-limited after 5 retries: {url}") } + /// Search `q` for up to ten pages of 100 results, pacing requests with + /// [`SEARCH_GAP`]. Record queries exceeding the search cap or marked + /// incomplete while returning the available hits. + /// + /// Items missing required string fields are omitted. Missing counts or + /// item arrays default to zero or empty. Fetch and JSON parse errors propagate. fn search_all(&mut self, q: &str) -> Result> { let mut items = Vec::new(); for page in 1..=10 { @@ -171,6 +189,22 @@ fn urlencode(s: &str) -> String { .collect() } +/// Find external repositories depending on the selected owners' packages. +/// +/// Use `opts.packages` when supplied; otherwise discover names from the owners' +/// `.ipkg` files. Match dependencies against the filtered catalogue and search +/// for owner references in `pack.toml` and `.gitmodules`. Exclude self accounts +/// case-insensitively and process each file URL once. +/// +/// Return one row per matching repository, ordered by confidence (high first) +/// then repository name, together with the filtered catalogue size and queries +/// whose results were truncated or incomplete. Network access and waits use `io`. +/// +/// # Errors +/// +/// Propagate I/O and search-response JSON errors, non-rate-limit HTTP failures, +/// and exhausted rate-limit retries. Invalid `pack.toml` files are skipped +/// instead of failing the scan. pub fn run(opts: &Options, io: &mut I) -> Result { let self_set: HashSet = opts .self_accounts From 258d4c658c952a66ce961bd572da072179f9bf88 Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:39:17 +0000 Subject: [PATCH 5/6] fix(idris-dependents): skip and report files returning HTTP 404, 410, or 451 --- idris-dependents/README.adoc | 3 + idris-dependents/src/main.rs | 6 ++ idris-dependents/src/scan.rs | 48 +++++++-- idris-dependents/tests/scan_tests.rs | 146 +++++++++++++++++++++++++-- 4 files changed, 187 insertions(+), 16 deletions(-) diff --git a/idris-dependents/README.adoc b/idris-dependents/README.adoc index 55f86306..81f829ac 100644 --- a/idris-dependents/README.adoc +++ b/idris-dependents/README.adoc @@ -41,6 +41,9 @@ cargo run -- parse-ipkg path/to/pkg.ipkg # offline: name + depends as JSON Queries that GitHub truncates (over 1000 results) or marks incomplete are listed in the report rather than silently dropped. +Files returning HTTP 404, 410, or 451 are logged and listed as `skippedUrls` +in the JSON report and in the CLI output; scanning continues. Other fetch +errors, exhausted rate-limit retries, and failed search responses abort the scan. == How the `.ipkg` reader is checked diff --git a/idris-dependents/src/main.rs b/idris-dependents/src/main.rs index f28e3cd8..d0a5ff4d 100644 --- a/idris-dependents/src/main.rs +++ b/idris-dependents/src/main.rs @@ -150,6 +150,12 @@ fn print_report(r: &Report) { r.incomplete_queries.join("; ") ); } + if !r.skipped_urls.is_empty() { + println!("\nSkipped unavailable files (HTTP 404, 410, or 451):"); + for url in &r.skipped_urls { + println!(" {url}"); + } + } } fn main() -> Result<()> { diff --git a/idris-dependents/src/scan.rs b/idris-dependents/src/scan.rs index 5963d52e..d0da2442 100644 --- a/idris-dependents/src/scan.rs +++ b/idris-dependents/src/scan.rs @@ -24,6 +24,15 @@ pub struct Response { pub body: String, } +impl Response { + fn into_body(self, url: &str) -> Result { + if !(200..300).contains(&self.status) { + bail!("HTTP {} for {url}", self.status); + } + Ok(self.body) + } +} + /// Side effects the scan needs. `raw` asks for file contents instead of JSON. pub trait Io { fn get(&mut self, url: &str, raw: bool) -> Result; @@ -59,6 +68,8 @@ pub struct Report { pub rows: Vec, /// Queries whose results GitHub truncated or marked incomplete. pub incomplete_queries: Vec, + /// Raw file URLs unavailable with HTTP 404, 410, or 451. + pub skipped_urls: Vec, } #[derive(Clone, Copy, PartialEq)] @@ -87,10 +98,11 @@ struct Acc { struct Scanner<'a, I: Io> { io: &'a mut I, incomplete: Vec, + skipped_urls: Vec, } impl Scanner<'_, I> { - fn get(&mut self, url: &str, raw: bool) -> Result { + fn get(&mut self, url: &str, raw: bool) -> Result { for _ in 0..=5 { let res = self.io.get(url, raw)?; let h = |k: &str| res.headers.get(k).map(String::as_str); @@ -98,10 +110,7 @@ impl Scanner<'_, I> { || (res.status == 403 && (h("retry-after").is_some() || h("x-ratelimit-remaining") == Some("0"))); if !limited { - if !(200..300).contains(&res.status) { - bail!("HTTP {} for {url}", res.status); - } - return Ok(res.body); + return Ok(res); } let retry_after = h("retry-after").and_then(|v| v.parse::().ok()); let reset = h("x-ratelimit-reset").and_then(|v| v.parse::().ok()); @@ -116,6 +125,19 @@ impl Scanner<'_, I> { bail!("still rate-limited after 5 retries: {url}") } + fn get_file(&mut self, url: &str) -> Result> { + let res = self.get(url, true)?; + if matches!(res.status, 404 | 410 | 451) { + self.io.log(&format!( + " skipping unavailable file {url}: HTTP {}", + res.status + )); + self.skipped_urls.push(url.to_string()); + return Ok(None); + } + res.into_body(url).map(Some) + } + fn search_all(&mut self, q: &str) -> Result> { let mut items = Vec::new(); for page in 1..=10 { @@ -123,8 +145,9 @@ impl Scanner<'_, I> { "{API}/search/code?q={}&per_page=100&page={page}", urlencode(q) ); - let body: serde_json::Value = serde_json::from_str(&self.get(&url, false)?) - .with_context(|| format!("search response for {q}"))?; + let body: serde_json::Value = + serde_json::from_str(&self.get(&url, false)?.into_body(&url)?) + .with_context(|| format!("search response for {q}"))?; self.io.sleep(SEARCH_GAP); let total = body["total_count"].as_u64().unwrap_or(0); if page == 1 && total > SEARCH_CAP { @@ -180,6 +203,7 @@ pub fn run(opts: &Options, io: &mut I) -> Result { let mut sc = Scanner { io, incomplete: Vec::new(), + skipped_urls: Vec::new(), }; // 1. Catalogue of your package names (or use --packages to skip this step). @@ -190,7 +214,10 @@ pub fn run(opts: &Options, io: &mut I) -> Result { for o in &opts.owners { sc.io.log(&format!("cataloguing .ipkg packages under {o}…")); for it in sc.search_all(&format!("package extension:ipkg user:{o}"))? { - if let Some(n) = ipkg::package_name(&sc.get(&it.url, true)?) { + let Some(text) = sc.get_file(&it.url)? else { + continue; + }; + if let Some(n) = ipkg::package_name(&text) { names.insert(n); } } @@ -230,7 +257,9 @@ pub fn run(opts: &Options, io: &mut I) -> Result { { continue; } - let text = sc.get(&it.url, true)?; + let Some(text) = sc.get_file(&it.url)? else { + continue; + }; let acc = hits.entry(it.full_name.clone()).or_default(); acc.owner = it.owner.clone(); let found = match kind { @@ -297,5 +326,6 @@ pub fn run(opts: &Options, io: &mut I) -> Result { catalogue_size: distinctive.len(), rows, incomplete_queries: sc.incomplete, + skipped_urls: sc.skipped_urls, }) } diff --git a/idris-dependents/tests/scan_tests.rs b/idris-dependents/tests/scan_tests.rs index fe195f11..5322c0b9 100644 --- a/idris-dependents/tests/scan_tests.rs +++ b/idris-dependents/tests/scan_tests.rs @@ -9,9 +9,11 @@ use idris_dependents::catalogue::Confidence; use idris_dependents::scan::{run, Io, Options, Response}; use serde_json::json; +#[derive(Default)] struct Mock { searches: BTreeMap<&'static str, Vec>, contents: BTreeMap<&'static str, &'static str>, + responses: BTreeMap<&'static str, Response>, limited_once: bool, logs: Vec, gets: usize, @@ -44,6 +46,13 @@ fn decode(q: &str) -> String { impl Io for Mock { fn get(&mut self, url: &str, raw: bool) -> Result { self.gets += 1; + if let Some(res) = self.responses.get(url) { + return Ok(Response { + status: res.status, + headers: res.headers.clone(), + body: res.body.clone(), + }); + } let ok = |body: String| Response { status: 200, headers: BTreeMap::new(), @@ -87,6 +96,7 @@ fn run_filters_self_dedups_files_classifies_survives_rate_limit() { vec![ item("alice/app", "app.ipkg", "c/alice"), item("alice/app", "app.ipkg", "c/alice"), // duplicate file + item("alice/app", "other.ipkg", "c/alice-other"), // same dependency item("bob/b", "b.ipkg", "c/bob"), item("hyperpolymath/x", "x.ipkg", "c/self"), item("dave/d", "d.ipkg", "c/dave"), @@ -100,6 +110,7 @@ fn run_filters_self_dedups_files_classifies_survives_rate_limit() { ]), contents: BTreeMap::from([ ("c/alice", "package app\ndepends = base, proven-dns"), + ("c/alice-other", "package other\ndepends = proven-dns"), ("c/bob", "package b\ndepends = proven"), // ambiguous -> LOW ( "c/carol-pack", @@ -115,6 +126,7 @@ fn run_filters_self_dedups_files_classifies_survives_rate_limit() { limited_once: false, logs: vec![], gets: 0, + responses: BTreeMap::new(), }; let opts = Options { owners: vec!["hyperpolymath".into()], @@ -141,18 +153,138 @@ fn run_filters_self_dedups_files_classifies_survives_rate_limit() { ("bob/b", Confidence::Low), ] ); + let alice = res.rows.iter().find(|r| r.repo == "alice/app").unwrap(); + assert_eq!(alice.deps, vec!["proven-dns"]); assert_eq!( - res.rows - .iter() - .find(|r| r.repo == "alice/app") - .unwrap() - .files - .len(), - 1 + alice.files, + vec![ + "https://github.com/alice/app/blob/main/app.ipkg", + "https://github.com/alice/app/blob/main/other.ipkg", + ] ); assert_eq!(res.catalogue_size, 3); assert!(io.logs.iter().any(|l| l.starts_with("rate-limited"))); assert!(res.incomplete_queries.is_empty()); + assert!(res.skipped_urls.is_empty()); +} + +fn catalogue_scan() -> (Options, Mock) { + let opts = Options { + owners: vec!["o".into()], + self_accounts: vec!["o".into()], + packages: None, + }; + let io = Mock { + searches: BTreeMap::from([ + ( + "package extension:ipkg user:o", + vec![ + item("o/pkg", "gone.ipkg", "c/catalogue-gone"), + item("o/pkg", "pkg.ipkg", "c/catalogue"), + ], + ), + ( + "ochrance extension:ipkg", + vec![ + item("alice/app", "gone.ipkg", "c/dependent-gone"), + item("alice/app", "app.ipkg", "c/dependent"), + ], + ), + ]), + contents: BTreeMap::from([ + ("c/catalogue-gone", "package ochrance"), + ("c/catalogue", "package ochrance"), + ("c/dependent-gone", "package gone\ndepends = ochrance"), + ("c/dependent", "package app\ndepends = ochrance"), + ]), + ..Mock::default() + }; + (opts, io) +} + +fn response(status: u16) -> Response { + Response { + status, + headers: BTreeMap::new(), + body: String::new(), + } +} + +#[test] +fn unavailable_files_are_logged_reported_and_skipped_in_both_raw_fetch_paths() { + for status in [404, 410, 451] { + let (opts, mut io) = catalogue_scan(); + io.responses = BTreeMap::from([ + ("c/catalogue-gone", response(status)), + ("c/dependent-gone", response(status)), + ]); + let report = run(&opts, &mut io).unwrap(); + assert_eq!(report.catalogue_size, 1); + assert_eq!(report.rows.len(), 1); + assert_eq!(report.rows[0].deps, vec!["ochrance"]); + assert_eq!( + report.rows[0].files, + vec!["https://github.com/alice/app/blob/main/app.ipkg"] + ); + assert_eq!( + report.skipped_urls, + vec!["c/catalogue-gone", "c/dependent-gone"] + ); + for url in &report.skipped_urls { + assert!(io + .logs + .iter() + .any(|l| l.contains(url) && l.contains(&format!("HTTP {status}")))); + } + assert_eq!( + serde_json::to_value(&report).unwrap()["skippedUrls"], + json!(["c/catalogue-gone", "c/dependent-gone"]) + ); + } +} + +#[test] +fn other_raw_errors_abort_in_both_fetch_paths() { + for url in ["c/catalogue-gone", "c/dependent-gone"] { + for status in [401, 403, 422, 429, 500, 503] { + let (opts, mut io) = catalogue_scan(); + io.responses.insert(url, response(status)); + let err = run(&opts, &mut io).unwrap_err().to_string(); + assert!(err.contains(url), "{err}"); + if status == 429 { + assert!(err.contains("still rate-limited"), "{err}"); + } else { + assert!(err.contains(&format!("HTTP {status}")), "{err}"); + } + } + let (opts, mut io) = catalogue_scan(); + let mut limited = response(403); + limited + .headers + .insert("x-ratelimit-remaining".into(), "0".into()); + io.responses.insert(url, limited); + let err = run(&opts, &mut io).unwrap_err().to_string(); + assert!( + err.contains("still rate-limited") && err.contains(url), + "{err}" + ); + } +} + +#[test] +fn unavailable_search_responses_and_invalid_json_abort() { + let url = "https://api.github.com/search/code?q=package%20extension%3Aipkg%20user%3Ao&per_page=100&page=1"; + for status in [404, 410, 451, 200] { + let (opts, mut io) = catalogue_scan(); + io.responses.insert(url, response(status)); + let err = run(&opts, &mut io).unwrap_err().to_string(); + if status == 200 { + assert!(err.contains("search response"), "{err}"); + } else { + assert!(err.contains(&format!("HTTP {status}")), "{err}"); + } + assert!(!io.logs.iter().any(|l| l.contains("skipping"))); + } } #[test] From e1585a3044359aafbdfd491b8053bc5a1417f597 Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:42:00 +0000 Subject: [PATCH 6/6] test(idris-dependents): expand parser, scan edge case, and offline CLI coverage --- idris-dependents/tests/cli_tests.rs | 161 ++++++++ idris-dependents/tests/rules_tests.rs | 268 ++++++++++++- idris-dependents/tests/scan_edge_tests.rs | 446 ++++++++++++++++++++++ idris-dependents/tests/scan_tests.rs | 247 ++++++++++++ 4 files changed, 1121 insertions(+), 1 deletion(-) create mode 100644 idris-dependents/tests/cli_tests.rs create mode 100644 idris-dependents/tests/scan_edge_tests.rs diff --git a/idris-dependents/tests/cli_tests.rs b/idris-dependents/tests/cli_tests.rs new file mode 100644 index 00000000..71b0c16e --- /dev/null +++ b/idris-dependents/tests/cli_tests.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Offline CLI contracts. Child processes never inherit GitHub credentials. + +use std::path::PathBuf; +use std::process::{Command, Output, Stdio}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use serde_json::{json, Value}; + +fn cli() -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_idris-dependents")); + command + .env_remove("GITHUB_TOKEN") + .env_remove("GH_TOKEN") + .stdin(Stdio::null()); + command +} + +fn fixture(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures") + .join(name) +} + +fn output_json(output: Output) -> Value { + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + output.stderr.is_empty(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + serde_json::from_slice(&output.stdout).expect("CLI stdout must be JSON") +} + +struct InputFile(PathBuf); + +impl InputFile { + fn new(contents: &str) -> Self { + static NEXT: AtomicUsize = AtomicUsize::new(0); + let dir = std::env::temp_dir().join(format!( + "idris-dependents-cli-{}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(), + NEXT.fetch_add(1, Ordering::Relaxed), + )); + std::fs::create_dir(&dir).unwrap(); + let file = Self(dir.join("input with spaces.toml")); + std::fs::write(&file.0, contents).unwrap(); + file + } +} + +impl Drop for InputFile { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(self.0.parent().unwrap()); + } +} + +#[test] +fn parse_ipkg_prints_the_package_and_dependencies_without_authentication() { + let output = cli() + .arg("parse-ipkg") + .arg(fixture("ochrance.ipkg")) + .output() + .unwrap(); + assert_eq!( + output_json(output), + json!({ + "package": "ochrance", "depends": ["base", "contrib", "network"], + }) + ); +} + +#[test] +fn parse_ipkg_with_no_declaration_prints_null_and_an_empty_dependency_list() { + let file = InputFile::new("-- package fake\n{- depends = fake -}"); + let output = cli().arg("parse-ipkg").arg(&file.0).output().unwrap(); + assert_eq!( + output_json(output), + json!({ "package": null, "depends": [] }) + ); +} + +#[test] +fn parse_pack_toml_honours_default_and_comma_delimited_owner_options() { + let file = InputFile::new( + r#" +[custom.all.first] +url = "https://github.com/hyperpolymath/first" +[custom.all.second] +url = "https://github.com/another/second" +"#, + ); + let first = json!({"name": "first", "url": "https://github.com/hyperpolymath/first"}); + let second = json!({"name": "second", "url": "https://github.com/another/second"}); + for (owners, expected) in [ + (None, json!([first.clone()])), + (Some("ANOTHER"), json!([second.clone()])), + (Some("hyperpolymath,another"), json!([first, second])), + (Some("nobody"), json!([])), + ] { + let mut command = cli(); + command.arg("parse-pack-toml").arg(&file.0); + if let Some(owners) = owners { + command.args(["--owners", owners]); + } + assert_eq!(output_json(command.output().unwrap()), expected); + } +} + +#[test] +fn offline_commands_report_missing_paths_and_invalid_toml_as_errors() { + let file = InputFile::new("[invalid table"); + for subcommand in ["parse-ipkg", "parse-pack-toml"] { + let missing = file.0.with_file_name("missing input"); + let output = cli().arg(subcommand).arg(&missing).output().unwrap(); + assert!(!output.status.success()); + assert!(output.stdout.is_empty()); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!( + stderr.contains(&format!("reading {}", missing.display())), + "{stderr}" + ); + } + let output = cli().arg("parse-pack-toml").arg(&file.0).output().unwrap(); + assert!(!output.status.success()); + assert!(output.stdout.is_empty()); + assert!(String::from_utf8_lossy(&output.stderr).contains("TOML parse error")); +} + +#[test] +fn scan_without_credentials_fails_before_searching_or_writing_a_report() { + let file = InputFile::new("preserve this output"); + for empty_token in [false, true] { + let mut command = cli(); + if empty_token { + command.env("GITHUB_TOKEN", "").env("GH_TOKEN", ""); + } + let output = command + .args(["scan", "--packages", "ochrance", "--json"]) + .arg(&file.0) + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(output.stdout.is_empty()); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains("set GITHUB_TOKEN"), "{stderr}"); + assert!(stderr.contains("code search requires auth"), "{stderr}"); + assert_eq!( + std::fs::read_to_string(&file.0).unwrap(), + "preserve this output" + ); + } +} diff --git a/idris-dependents/tests/rules_tests.rs b/idris-dependents/tests/rules_tests.rs index 3ed461cc..a4d06633 100644 --- a/idris-dependents/tests/rules_tests.rs +++ b/idris-dependents/tests/rules_tests.rs @@ -3,7 +3,7 @@ //! plus regressions for bugs found in that implementation (P1–P6). use idris_dependents::catalogue::{classify, is_distinctive, search_stems, Confidence}; -use idris_dependents::ipkg::{depends, package_name}; +use idris_dependents::ipkg::{depends, lex, package_name, Tok}; use idris_dependents::refs::{gitmodules_refs, pack_toml_refs, points_at}; fn fx(name: &str) -> String { @@ -304,3 +304,269 @@ fn gitmodules_quoted_and_crlf() { s(&["https://github.com/hyperpolymath/proven"]) ); } + +#[test] +fn lexer_distinguishes_field_assignment_from_every_version_operator() { + assert_eq!( + lex("= == < <= > >= && , . 012"), + ( + vec![ + Tok::Equals, + Tok::EqOp, + Tok::Lt, + Tok::Lte, + Tok::Gt, + Tok::Gte, + Tok::AndOp, + Tok::Sep, + Tok::Dot, + Tok::Int("012".into()), + ], + 0, + ) + ); +} + +#[test] +fn lexer_preserves_identifiers_strings_and_counts_only_unrecognised_characters() { + let (tokens, skipped) = lex("\0@$ _pkg' café-lib Data.Module \"a\\\"--{-b\""); + assert_eq!(skipped, 3); + assert_eq!( + tokens, + vec![ + Tok::Ident { + text: "_pkg'".into(), + dotted: false + }, + Tok::Ident { + text: "café-lib".into(), + dotted: false + }, + Tok::Ident { + text: "Data.Module".into(), + dotted: true + }, + Tok::Str("a\\\"--{-b".into()), + ] + ); +} + +#[test] +fn fake_declarations_in_comments_strings_and_dotted_names_are_ignored() { + let text = r#"-- package fake +{- package hidden -} +opts = "package quoted depends = bogus \" depends = escaped" +package Wrong.Module +package real +depends = café-lib, _ffi', café-lib +"#; + assert_eq!(package_name(text).as_deref(), Some("real")); + assert_eq!(depends(text), s(&["café-lib", "_ffi'"])); +} + +#[test] +fn depends_requires_an_exact_field_key_and_single_equals() { + for text in [ + "mydepends = ochrance", + "Depends = ochrance", + "Project.depends = ochrance", + "depends == ochrance", + "depends >= ochrance", + "depends", + "depends =", + "depends = 123", + ] { + assert!(depends(text).is_empty(), "{text}"); + } +} + +#[test] +fn all_version_bounds_preserve_dependency_order_and_stop_at_fields() { + assert_eq!( + depends("depends = a < 2, b <= 2.1, c > 0, d >= 1.0, e == 3, f >= 1 && < 2,\nmodules = M\ndepends = e, g"), + s(&["a", "b", "c", "d", "e", "f", "g"]) + ); +} + +#[test] +fn unterminated_block_comment_preserves_only_preceding_dependencies() { + let text = "package real\ndepends = ochrance\n{- unfinished {- nested -}\ndepends = fake"; + assert_eq!(package_name(text).as_deref(), Some("real")); + assert_eq!(depends(text), s(&["ochrance"])); + assert!(depends("{- depends = fake").is_empty()); +} + +#[test] +fn empty_and_comment_only_files_have_no_package_or_dependencies() { + for text in ["", " \t\r\n", "-- package fake\r\n", "{- depends = fake -}"] { + assert_eq!(package_name(text), None, "{text}"); + assert!(depends(text).is_empty(), "{text}"); + } +} + +#[test] +fn internal_package_filter_is_case_insensitive_and_respects_affix_boundaries() { + for name in [ + "ABI", + "Benchmark", + "BENCHMARKS", + "Test", + "TESTS", + "CoreOnly", + "FFI-Test", + "Integration-Tests", + "Property-Tests", + "TEST-helper", + "Pkg-TEST", + "Pkg-TESTS", + "Pkg-BENCH", + "Pkg-SUITE", + ] { + assert!(!is_distinctive(name), "{name}"); + } + for name in [ + "test-", + "testing", + "contest", + "benchmarker", + "suite", + "abi-tools", + ] { + assert!(is_distinctive(name), "{name}"); + } +} + +#[test] +fn search_stem_boundary_counts_characters_and_returns_sorted_unique_names() { + assert_eq!( + search_stems(&s(&[ + "ééééé-core", + "éééé-core", + "abcde-two", + "abcd-two", + "abcde-one", + "abcde", + "abcde-tests", + "TEST-helper", + "abcd-two", + ])), + s(&["abcd-two", "abcde", "éééé-core", "ééééé"]) + ); + assert!(search_stems(&s(&["test", "abi", "pkg-bench"])).is_empty()); +} + +#[test] +fn classification_accepts_submodules_and_promotes_mixed_evidence() { + for (deps, pack, sub, expected) in [ + (s(&[]), 0, 1, Some(Confidence::High)), + (s(&["proven", "trope"]), 0, 0, Some(Confidence::Low)), + (s(&["proven", "ochrance"]), 0, 0, Some(Confidence::Medium)), + (s(&["trope"]), 1, 0, Some(Confidence::High)), + (s(&["ochrance"]), 0, 1, Some(Confidence::High)), + (s(&[]), 0, 0, None), + ] { + assert_eq!( + classify(&deps, pack, sub), + expected, + "{deps:?}, {pack}, {sub}" + ); + } +} + +#[test] +fn github_references_require_owner_and_host_boundaries() { + for url in [ + "HTTPS://GITHUB.COM/HyperPolymath/repo.git", + "git@github.com:METADATASTICIAN/repo.git", + "ssh://git@github.com/hyperpolymath/repo.git", + ] { + assert!(points_at(url, &own()), "{url}"); + assert!(!points_at(url, &[]), "empty owner list: {url}"); + } + for url in [ + "https://github.com/hyperpolymath-extra/repo", + "https://github.com/not-hyperpolymath/repo", + "https://github.com/hyperpolymath", + "https://github.com.evil.test/hyperpolymath/repo", + "https://github.com@evil.test/hyperpolymath/repo", + "https://notgithub.com/hyperpolymath/repo", + "../hyperpolymath/repo", + "", + ] { + assert!(!points_at(url, &own()), "{url}"); + } +} + +#[test] +fn pack_refs_keep_collection_and_entry_order_and_skip_irrelevant_shapes() { + let text = r#" +[package] +url = "https://github.com/hyperpolymath/not-custom" +[custom.z] +scalar = 3 +wrong-type = { url = 42 } +local = { path = "." } +external = { url = "https://github.com/someone/else" } +z-last-alphabetically = { url = 'https://github.com/hyperpolymath/first' } +a-first-alphabetically = { url = "git@github.com:metadatastician/second.git" } +[custom.a.third] +url = "https://github.com/hyperpolymath/third#readme" +"#; + let refs = pack_toml_refs(text, &own()).unwrap(); + assert_eq!( + refs.iter() + .map(|r| (r.name.as_str(), r.url.as_str())) + .collect::>(), + vec![ + ( + "z-last-alphabetically", + "https://github.com/hyperpolymath/first" + ), + ( + "a-first-alphabetically", + "git@github.com:metadatastician/second.git" + ), + ("third", "https://github.com/hyperpolymath/third#readme"), + ] + ); + assert!(pack_toml_refs(text, &[]).unwrap().is_empty()); +} + +#[test] +fn pack_refs_reject_invalid_toml_and_tolerate_absent_custom_tables() { + for text in ["[custom", "url = 'unterminated", "custom = 1\ncustom = 2"] { + assert!(pack_toml_refs(text, &own()).is_err(), "{text}"); + } + for text in [ + "", + "custom = 1", + "custom = []", + "[custom]\ncollection = 7", + "[package]\nname = 'x'", + ] { + assert!(pack_toml_refs(text, &own()).unwrap().is_empty(), "{text}"); + } +} + +#[test] +fn gitmodules_url_keys_comments_and_quoted_fragments() { + let text = r#" +# url = https://github.com/hyperpolymath/comment +; url = https://github.com/hyperpolymath/comment +url-extra = https://github.com/hyperpolymath/not-a-url-key +path = https://github.com/hyperpolymath/not-a-url +url = https://github.com/hyperpolymath/first.git ; trailing comment +url = https://github.com/metadatastician/second.git # trailing comment +url = "https://github.com/hyperpolymath/third#fragment;part" # comment +url = https://github.com/hyperpolymathx/not-ours +url = "" +"#; + assert_eq!( + gitmodules_refs(text, &own()), + s(&[ + "https://github.com/hyperpolymath/first.git", + "https://github.com/metadatastician/second.git", + "https://github.com/hyperpolymath/third#fragment;part", + ]) + ); +} diff --git a/idris-dependents/tests/scan_edge_tests.rs b/idris-dependents/tests/scan_edge_tests.rs new file mode 100644 index 00000000..587bbc37 --- /dev/null +++ b/idris-dependents/tests/scan_edge_tests.rs @@ -0,0 +1,446 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Deterministic scan boundaries: requests, pagination and failure handling. + +use std::collections::VecDeque; +use std::time::Duration; + +use anyhow::{anyhow, Result}; +use idris_dependents::catalogue::Confidence; +use idris_dependents::scan::{run, Io, Options, Response, API, SEARCH_GAP}; +use serde_json::{json, Value}; + +const QUERY: &str = "ochrance%20extension%3Aipkg"; + +struct Request { + url: String, + raw: bool, + response: Result, +} + +#[derive(Default)] +struct Script { + requests: VecDeque, + sleeps: Vec, + logs: Vec, +} + +impl Script { + fn respond(&mut self, url: &str, raw: bool, response: Result) { + self.requests.push_back(Request { + url: url.into(), + raw, + response, + }); + } + + // Queries are encoded explicitly by callers, independently of the production encoder. + fn search(&mut self, query: &str, page: u32, total: u64, incomplete: bool, items: Vec) { + self.respond( + &search_url(query, page), + false, + Ok(response( + 200, + &[], + &json!({ + "total_count": total, "incomplete_results": incomplete, "items": items, + }) + .to_string(), + )), + ); + } + + fn raw(&mut self, url: &str, text: &str) { + self.respond(url, true, Ok(response(200, &[], text))); + } + + fn done(&self) { + assert!( + self.requests.is_empty(), + "{} requests not made", + self.requests.len() + ); + } +} + +impl Io for Script { + fn get(&mut self, url: &str, raw: bool) -> Result { + let request = self.requests.pop_front().expect("unexpected GET"); + assert_eq!(url, request.url); + assert_eq!(raw, request.raw, "wrong response format for {url}"); + request.response + } + + fn sleep(&mut self, duration: Duration) { + self.sleeps.push(duration); + } + + fn now_secs(&self) -> u64 { + 100 + } + + fn log(&mut self, message: &str) { + self.logs.push(message.into()); + } +} + +fn search_url(query: &str, page: u32) -> String { + format!("{API}/search/code?q={query}&per_page=100&page={page}") +} + +fn response(status: u16, headers: &[(&str, &str)], body: &str) -> Response { + Response { + status, + headers: headers + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + body: body.into(), + } +} + +fn options() -> Options { + Options { + owners: vec![], + self_accounts: vec![], + packages: Some(vec!["ochrance".into()]), + } +} + +fn item(repo: &str, path: &str, url: &str) -> Value { + json!({ + "url": url, + "html_url": format!("https://github.com/{repo}/blob/main/{path}"), + "repository": { "full_name": repo, "owner": { "login": repo.split('/').next().unwrap() } }, + }) +} + +#[test] +fn pagination_fetches_the_second_page_and_reads_each_file_once() { + let mut io = Script::default(); + let first = item("alice/app", "a.ipkg", "contents/a"); + let last = item("bob/app", "b.ipkg", "contents/b"); + io.search(QUERY, 1, 101, false, vec![first; 100]); + io.search(QUERY, 2, 101, false, vec![last]); + io.raw("contents/a", "package a\ndepends = ochrance"); + io.raw("contents/b", "package b\ndepends = ochrance"); + + let report = run(&options(), &mut io).unwrap(); + assert_eq!( + report + .rows + .iter() + .map(|r| r.repo.as_str()) + .collect::>(), + vec!["alice/app", "bob/app"] + ); + assert!(report.rows.iter().all(|r| r.files.len() == 1)); + assert!(report.incomplete_queries.is_empty()); + assert_eq!(io.sleeps, vec![SEARCH_GAP; 2]); + io.done(); +} + +#[test] +fn a_full_page_stops_when_total_count_is_reached() { + let mut io = Script::default(); + io.search( + QUERY, + 1, + 100, + false, + vec![item("alice/app", "a.ipkg", "contents/a"); 100], + ); + io.raw("contents/a", "depends = ochrance"); + let report = run(&options(), &mut io).unwrap(); + assert_eq!(report.rows.len(), 1); + assert!(report.incomplete_queries.is_empty()); + assert_eq!(io.sleeps, vec![SEARCH_GAP]); + io.done(); +} + +#[test] +fn pagination_caps_at_ten_pages_and_only_totals_above_1000_are_truncated() { + for total in [1000, 1001] { + let mut io = Script::default(); + for page in 1..=10 { + io.search( + QUERY, + page, + total, + false, + vec![item("alice/app", "a.ipkg", "contents/a"); 100], + ); + } + io.raw("contents/a", "depends = ochrance"); + let report = run(&options(), &mut io).unwrap(); + assert_eq!(report.rows.len(), 1); + assert_eq!( + report.incomplete_queries, + if total > 1000 { + vec!["ochrance extension:ipkg"] + } else { + vec![] + } + ); + assert_eq!(io.sleeps, vec![SEARCH_GAP; 10]); + io.done(); + } +} + +#[test] +fn incomplete_flag_on_later_pages_is_reported_once_even_with_truncation() { + for total in [201, 1001] { + let mut io = Script::default(); + let hit = item("alice/app", "a.ipkg", "contents/a"); + io.search(QUERY, 1, total, false, vec![hit.clone(); 100]); + io.search(QUERY, 2, total, true, vec![hit.clone(); 100]); + io.search(QUERY, 3, total, true, vec![hit]); + io.raw("contents/a", "depends = ochrance"); + let report = run(&options(), &mut io).unwrap(); + assert_eq!(report.incomplete_queries, vec!["ochrance extension:ipkg"]); + assert_eq!(report.rows.len(), 1); + io.done(); + } +} + +#[test] +fn incomplete_empty_search_still_appears_in_the_report() { + let mut io = Script::default(); + io.search(QUERY, 1, 0, true, vec![]); + let report = run(&options(), &mut io).unwrap(); + assert!(report.rows.is_empty()); + assert_eq!(report.incomplete_queries, vec!["ochrance extension:ipkg"]); + io.done(); +} + +#[test] +fn rate_limit_waits_use_retry_after_then_reset_then_fallback() { + let cases = [ + (429, vec![], 60), + ( + 429, + vec![("retry-after", "7"), ("x-ratelimit-reset", "999")], + 7, + ), + ( + 403, + vec![("x-ratelimit-remaining", "0"), ("x-ratelimit-reset", "120")], + 21, + ), + ( + 403, + vec![("retry-after", "0"), ("x-ratelimit-reset", "90")], + 2, + ), + ( + 403, + vec![("retry-after", "bad"), ("x-ratelimit-reset", "100")], + 2, + ), + ( + 429, + vec![("retry-after", "bad"), ("x-ratelimit-reset", "bad")], + 60, + ), + (403, vec![("retry-after", "1")], 1), + ]; + for (status, headers, wait) in cases { + let mut io = Script::default(); + io.respond( + &search_url(QUERY, 1), + false, + Ok(response(status, &headers, "")), + ); + io.search(QUERY, 1, 0, false, vec![]); + assert!(run(&options(), &mut io).unwrap().rows.is_empty()); + assert_eq!( + io.sleeps, + vec![Duration::from_secs(wait), SEARCH_GAP], + "status {status}, headers {headers:?}" + ); + assert!(io + .logs + .iter() + .any(|s| s == &format!("rate-limited; waiting {wait}s"))); + io.done(); + } +} + +#[test] +fn persistent_rate_limits_stop_after_five_retries() { + let mut io = Script::default(); + for _ in 0..6 { + io.respond( + &search_url(QUERY, 1), + false, + Ok(response(429, &[("retry-after", "1")], "")), + ); + } + let error = run(&options(), &mut io).unwrap_err().to_string(); + assert!( + error.contains("still rate-limited after 5 retries"), + "{error}" + ); + assert!(error.contains(&search_url(QUERY, 1)), "{error}"); + io.done(); +} + +#[test] +fn hard_http_errors_are_not_retried_even_with_retry_headers() { + for status in [401, 403, 404, 500] { + let mut io = Script::default(); + // Reset alone does not make a 403 a rate-limit response. + io.respond( + &search_url(QUERY, 1), + false, + Ok(response(status, &[("x-ratelimit-reset", "120")], "")), + ); + let error = run(&options(), &mut io).unwrap_err().to_string(); + assert!(error.contains(&format!("HTTP {status}")), "{error}"); + assert!(error.contains(&search_url(QUERY, 1)), "{error}"); + assert!(io.sleeps.is_empty()); + io.done(); + } +} + +#[test] +fn transport_errors_are_propagated_without_retry() { + let mut io = Script::default(); + io.respond( + &search_url(QUERY, 1), + false, + Err(anyhow!("synthetic connection failure")), + ); + let error = run(&options(), &mut io).unwrap_err().to_string(); + assert!(error.contains("synthetic connection failure"), "{error}"); + assert!(io.sleeps.is_empty()); + io.done(); +} + +#[test] +fn malformed_search_json_identifies_the_query() { + let mut io = Script::default(); + io.respond( + &search_url(QUERY, 1), + false, + Ok(response(200, &[], "not JSON")), + ); + let error = run(&options(), &mut io).unwrap_err(); + assert!( + error + .to_string() + .contains("search response for ochrance extension:ipkg"), + "{error:#}" + ); + assert!(io.sleeps.is_empty()); + io.done(); +} + +#[test] +fn search_hits_missing_required_fields_are_skipped() { + let valid = item("alice/app", "a.ipkg", "contents/a"); + let mut items = Vec::new(); + for pointer in [ + "/url", + "/html_url", + "/repository/full_name", + "/repository/owner/login", + ] { + let mut broken = valid.clone(); + *broken.pointer_mut(pointer).unwrap() = Value::Null; + items.push(broken); + } + items.push(valid); + let mut io = Script::default(); + io.search(QUERY, 1, items.len() as u64, false, items); + io.raw("contents/a", "depends = ochrance"); + let report = run(&options(), &mut io).unwrap(); + assert_eq!(report.rows.len(), 1); + assert_eq!(report.rows[0].repo, "alice/app"); + io.done(); +} + +#[test] +fn raw_content_fetches_retry_rate_limits_and_propagate_hard_failures() { + let mut io = Script::default(); + io.search( + QUERY, + 1, + 1, + false, + vec![item("alice/app", "a.ipkg", "contents/a")], + ); + io.respond( + "contents/a", + true, + Ok(response(429, &[("retry-after", "3")], "")), + ); + io.raw("contents/a", "depends = ochrance"); + assert_eq!(run(&options(), &mut io).unwrap().rows.len(), 1); + assert_eq!(io.sleeps, vec![SEARCH_GAP, Duration::from_secs(3)]); + io.done(); + + let mut io = Script::default(); + io.search( + QUERY, + 1, + 1, + false, + vec![item("alice/app", "a.ipkg", "contents/a")], + ); + io.respond("contents/a", true, Ok(response(404, &[], ""))); + let error = run(&options(), &mut io).unwrap_err().to_string(); + assert!(error.contains("HTTP 404 for contents/a"), "{error}"); + io.done(); +} + +#[test] +fn queries_percent_encode_unicode_and_reserved_characters() { + let mut io = Script::default(); + let mut opts = options(); + opts.packages = Some(vec!["café+~_".into()]); + io.search("caf%C3%A9%2B~_%20extension%3Aipkg", 1, 0, false, vec![]); + assert!(run(&opts, &mut io).unwrap().rows.is_empty()); + io.done(); +} + +#[test] +fn explicit_empty_catalogue_makes_no_requests_when_no_owners_are_watched() { + let mut io = Script::default(); + let mut opts = options(); + opts.packages = Some(vec![]); + let report = run(&opts, &mut io).unwrap(); + assert_eq!(report.catalogue_size, 0); + assert!(report.rows.is_empty()); + assert!(report.incomplete_queries.is_empty()); + assert!(io.sleeps.is_empty()); + io.done(); +} + +#[test] +fn report_json_keeps_public_field_names_and_confidence_spelling() { + let mut io = Script::default(); + io.search( + QUERY, + 1, + 1, + false, + vec![item("alice/app", "a.ipkg", "contents/a")], + ); + io.raw("contents/a", "depends = ochrance"); + let report = run(&options(), &mut io).unwrap(); + assert_eq!(report.rows[0].confidence, Confidence::Medium); + assert_eq!( + serde_json::to_value(&report).unwrap(), + json!({ + "catalogueSize": 1, + "rows": [{ + "owner": "alice", "repo": "alice/app", "deps": ["ochrance"], + "packRefs": [], "subRefs": [], + "files": ["https://github.com/alice/app/blob/main/a.ipkg"], + "confidence": "MEDIUM", + }], + "incompleteQueries": [], + }) + ); + io.done(); +} diff --git a/idris-dependents/tests/scan_tests.rs b/idris-dependents/tests/scan_tests.rs index 5322c0b9..81958f19 100644 --- a/idris-dependents/tests/scan_tests.rs +++ b/idris-dependents/tests/scan_tests.rs @@ -347,3 +347,250 @@ fn non_rate_limit_errors_fail_loudly() { let err = run(&opts, &mut Forbidden).unwrap_err().to_string(); assert!(err.contains("HTTP 401"), "{err}"); } + +fn ready_mock() -> Mock { + Mock { + searches: BTreeMap::new(), + contents: BTreeMap::new(), + limited_once: true, + logs: vec![], + gets: 0, + } +} + +#[test] +fn catalogue_discovers_names_deduplicates_and_excludes_internal_packages() { + let mut io = ready_mock(); + io.searches.insert( + "package extension:ipkg user:hyperpolymath", + vec![ + item("hyperpolymath/a", "a.ipkg", "catalogue/a"), + item("hyperpolymath/b", "b.ipkg", "catalogue/b"), + item("hyperpolymath/c", "c.ipkg", "catalogue/c"), + item("hyperpolymath/d", "d.ipkg", "catalogue/d"), + ], + ); + io.searches.insert( + "ochrance extension:ipkg", + vec![item("alice/app", "app.ipkg", "external/app")], + ); + io.contents = BTreeMap::from([ + ("catalogue/a", "package ochrance\r\ndepends = base"), + ("catalogue/b", "package ochrance"), // same name in another repository + ("catalogue/c", "package ochrance-tests"), + ( + "catalogue/d", + "-- package fake\nmodules = MissingDeclaration", + ), + ("external/app", "depends = base, ochrance-tests, ochrance"), + ]); + let report = run( + &Options { + owners: vec!["hyperpolymath".into()], + self_accounts: vec!["hyperpolymath".into()], + packages: None, + }, + &mut io, + ) + .unwrap(); + assert_eq!(report.catalogue_size, 1); + assert_eq!(report.rows.len(), 1); + assert_eq!(report.rows[0].repo, "alice/app"); + assert_eq!(report.rows[0].deps, vec!["ochrance"]); + assert_eq!(report.rows[0].confidence, Confidence::Medium); + // 1 catalogue query + 4 catalogue files + 3 dependent queries + 1 external file. + assert_eq!(io.gets, 9); +} + +#[test] +fn scan_merges_all_evidence_and_sorts_by_confidence_then_repository() { + let mut io = ready_mock(); + let shared = item("zed/app", "one.ipkg", "content/one"); + io.searches = BTreeMap::from([ + ( + "ochrance extension:ipkg", + vec![ + shared.clone(), + item("zed/app", "two.ipkg", "content/two"), + item("zed/app", "noise.ipkg", "content/noise"), + item("carol/app", "app.ipkg", "content/carol"), + ], + ), + ( + "proven extension:ipkg", + vec![shared, item("bob/app", "app.ipkg", "content/bob")], + ), + ( + "hyperpolymath filename:pack.toml", + vec![item("zed/app", "pack.toml", "content/pack")], + ), + ( + "hyperpolymath filename:.gitmodules", + vec![ + item("zed/app", ".gitmodules", "content/sub"), + item("anna/app", ".gitmodules", "content/anna-sub"), + ], + ), + ]); + io.contents = BTreeMap::from([ + ( + "content/one", + "depends = proven-dns, ochrance, base, proven-tests", + ), + ("content/two", "depends = proven, ochrance, proven-dns"), + ( + "content/noise", + "-- depends = ochrance\ndepends = ochrance-extra, base", + ), + ("content/carol", "depends = ochrance"), + ("content/bob", "depends = proven"), + ( + "content/pack", + "[custom.all.local-name]\nurl = 'https://github.com/hyperpolymath/proven'", + ), + ( + "content/sub", + "url = git@github.com:hyperpolymath/ochrance.git", + ), + ( + "content/anna-sub", + "url = https://github.com/hyperpolymath/proven.git", + ), + ]); + let report = run( + &Options { + owners: vec!["hyperpolymath".into()], + self_accounts: vec![], + packages: Some(vec![ + "proven".into(), + "proven-dns".into(), + "ochrance".into(), + "proven-tests".into(), + ]), + }, + &mut io, + ) + .unwrap(); + assert_eq!( + report + .rows + .iter() + .map(|r| (r.repo.as_str(), r.confidence)) + .collect::>(), + vec![ + ("anna/app", Confidence::High), + ("zed/app", Confidence::High), + ("carol/app", Confidence::Medium), + ("bob/app", Confidence::Low), + ] + ); + let zed = &report.rows[1]; + assert_eq!(zed.owner, "zed"); + assert_eq!(zed.deps, vec!["ochrance", "proven", "proven-dns"]); + assert_eq!( + zed.pack_refs, + vec![idris_dependents::refs::PackRef { + name: "local-name".into(), + url: "https://github.com/hyperpolymath/proven".into(), + }] + ); + assert_eq!( + zed.sub_refs, + vec!["git@github.com:hyperpolymath/ochrance.git"] + ); + assert_eq!( + zed.files, + vec![ + "https://github.com/zed/app/blob/main/one.ipkg", + "https://github.com/zed/app/blob/main/two.ipkg", + "https://github.com/zed/app/blob/main/pack.toml", + "https://github.com/zed/app/blob/main/.gitmodules", + ] + ); + assert_eq!(report.catalogue_size, 3); + assert_eq!(io.gets, 12); // 4 searches + 8 unique files, including the irrelevant file +} + +#[test] +fn self_accounts_are_case_insensitive_exact_matches_and_skipped_before_fetch() { + let mut io = ready_mock(); + io.searches.insert( + "ochrance extension:ipkg", + vec![ + item("HyperPolymath/private", "a.ipkg", "must-not-fetch/self"), + item("ANOTHER/account", "a.ipkg", "must-not-fetch/alias"), + item("hyperpolymath-community/app", "a.ipkg", "external/app"), + ], + ); + io.contents.insert("external/app", "depends = ochrance"); + let report = run( + &Options { + owners: vec![], + self_accounts: vec!["hyperpolymath".into(), "another".into()], + packages: Some(vec!["ochrance".into()]), + }, + &mut io, + ) + .unwrap(); + assert_eq!(report.rows.len(), 1); + assert_eq!(report.rows[0].repo, "hyperpolymath-community/app"); + assert_eq!(io.gets, 2); +} + +#[test] +fn malformed_pack_files_are_logged_without_losing_other_evidence() { + let mut io = ready_mock(); + io.searches = BTreeMap::from([ + ( + "ochrance extension:ipkg", + vec![item("alice/app", "a.ipkg", "content/dep")], + ), + ( + "hyperpolymath filename:pack.toml", + vec![ + item("alice/app", "pack.toml", "content/bad"), + item("bob/app", "pack.toml", "content/only-bad"), + item("carol/app", "pack.toml", "content/unrelated"), + ], + ), + ]); + io.contents = BTreeMap::from([ + ("content/dep", "depends = ochrance"), + ("content/bad", "[custom.invalid"), + ("content/only-bad", "url = 'unfinished"), + ( + "content/unrelated", + "[custom.all.p]\nurl = 'https://github.com/other/ochrance'", + ), + ]); + let report = run( + &Options { + owners: vec!["hyperpolymath".into()], + self_accounts: vec![], + packages: Some(vec!["ochrance".into()]), + }, + &mut io, + ) + .unwrap(); + assert_eq!(report.rows.len(), 1); + let row = &report.rows[0]; + assert_eq!(row.repo, "alice/app"); + assert_eq!(row.confidence, Confidence::Medium); + assert!(row.pack_refs.is_empty()); + assert_eq!( + row.files, + vec!["https://github.com/alice/app/blob/main/a.ipkg"] + ); + let skips: Vec<_> = io + .logs + .iter() + .filter(|s| s.contains("skipping unparseable pack.toml")) + .collect(); + assert_eq!(skips.len(), 2); + assert!(skips + .iter() + .any(|s| s.contains("https://github.com/alice/app/blob/main/pack.toml"))); + assert!(skips + .iter() + .any(|s| s.contains("https://github.com/bob/app/blob/main/pack.toml"))); +}