diff --git a/.gitignore b/.gitignore index eeace22..915b2e0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ out/ report.md report.json *.exe +!tests/fixtures/report_contracts/**/report.md +!tests/fixtures/report_contracts/**/report.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 482a53a..8fa8e2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All notable user-visible changes should be recorded here. ### Added -- None yet. +- Added sanitized golden `report.md` / `report.json` regression fixtures to lock report contracts. ### Changed diff --git a/CMakeLists.txt b/CMakeLists.txt index 79fd0e6..8f2f14a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,4 +42,12 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/assets/sample_config.json ${CMAKE_CURRENT_BINARY_DIR}/cli_test_output ) + + add_executable(test_report_contracts tests/test_report_contracts.cpp) + add_test( + NAME report_contracts + COMMAND test_report_contracts + $ + ${CMAKE_CURRENT_BINARY_DIR}/report_contract_output + ) endif() diff --git a/tests/fixtures/report_contracts/journalctl_short_full/input.log b/tests/fixtures/report_contracts/journalctl_short_full/input.log new file mode 100644 index 0000000..6927cef --- /dev/null +++ b/tests/fixtures/report_contracts/journalctl_short_full/input.log @@ -0,0 +1,16 @@ +Tue 2026-03-10 08:11:22 UTC example-host sshd[2234]: Failed password for invalid user admin from 203.0.113.10 port 51022 ssh2 +Tue 2026-03-10 08:12:05 UTC example-host sshd[2235]: Failed password for root from 203.0.113.10 port 51030 ssh2 +Tue 2026-03-10 08:13:10 UTC example-host sshd[2236]: Failed password for test from 203.0.113.10 port 51040 ssh +Tue 2026-03-10 08:14:44 UTC example-host sshd[2237]: Failed password for guest from 203.0.113.10 port 51050 ssh2 +Tue 2026-03-10 08:18:05 UTC example-host sshd[2238]: Failed publickey for invalid user deploy from 203.0.113.10 port 51060 ssh2 +Tue 2026-03-10 08:20:10 UTC example-host sshd[2240]: Accepted password for alice from 203.0.113.20 port 51111 ssh2 +Tue 2026-03-10 08:21:00 UTC example-host sudo: alice : TTY=pts/0 ; PWD=/home/alice ; USER=root ; COMMAND=/usr/bin/systemctl restart ssh +Tue 2026-03-10 08:22:10 UTC example-host sudo: alice : TTY=pts/0 ; PWD=/home/alice ; USER=root ; COMMAND=/usr/bin/journalctl -xe +Tue 2026-03-10 08:24:15 UTC example-host sudo: alice : TTY=pts/0 ; PWD=/home/alice ; USER=root ; COMMAND=/usr/bin/vi /etc/ssh/sshd_config +Tue 2026-03-10 08:25:30 UTC example-host sshd[2241]: Failed password for bob from 203.0.113.30 port 51234 ssh2 +Tue 2026-03-10 08:26:02 UTC example-host sshd[2242]: Invalid user backup from 203.0.113.31 port 51236 +Tue 2026-03-10 08:28:33 UTC example-host pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=203.0.113.41 user=alice +Tue 2026-03-10 08:29:50 UTC example-host pam_unix(sudo:session): session opened for user root by alice(uid=0) +Tue 2026-03-10 08:30:12 UTC example-host sshd[2244]: Failed password for invalid user qauser from 203.0.113.50 port 51290 ssh2 +Tue 2026-03-10 08:31:18 UTC example-host sshd[2245]: Connection closed by authenticating user alice 203.0.113.51 port 51291 [preauth] +Tue 2026-03-10 08:32:26 UTC example-host sshd[2246]: Timeout, client not responding from 203.0.113.52 port 51292 diff --git a/tests/fixtures/report_contracts/journalctl_short_full/report.json b/tests/fixtures/report_contracts/journalctl_short_full/report.json new file mode 100644 index 0000000..abffa2e --- /dev/null +++ b/tests/fixtures/report_contracts/journalctl_short_full/report.json @@ -0,0 +1,64 @@ +{ + "tool": "LogLens", + "input": "tests/fixtures/report_contracts/journalctl_short_full/input.log", + "input_mode": "journalctl_short_full", + "timezone_present": true, + "parser_quality": { + "total_lines": 16, + "parsed_lines": 14, + "unparsed_lines": 2, + "parse_success_rate": 0.8750, + "top_unknown_patterns": [ + {"pattern": "sshd_connection_closed_preauth", "count": 1}, + {"pattern": "sshd_timeout_or_disconnection", "count": 1} + ] + }, + "parsed_event_count": 14, + "warning_count": 2, + "finding_count": 3, + "event_counts": [ + {"event_type": "ssh_failed_password", "count": 4}, + {"event_type": "ssh_accepted_password", "count": 1}, + {"event_type": "ssh_invalid_user", "count": 3}, + {"event_type": "ssh_failed_publickey", "count": 1}, + {"event_type": "pam_auth_failure", "count": 1}, + {"event_type": "session_opened", "count": 1}, + {"event_type": "sudo_command", "count": 3} + ], + "findings": [ + { + "rule": "brute_force", + "subject_kind": "source_ip", + "subject": "203.0.113.10", + "event_count": 5, + "window_start": "2026-03-10 08:11:22", + "window_end": "2026-03-10 08:18:05", + "usernames": [], + "summary": "5 failed SSH attempts from 203.0.113.10 within 10 minutes." + }, + { + "rule": "multi_user_probing", + "subject_kind": "source_ip", + "subject": "203.0.113.10", + "event_count": 5, + "window_start": "2026-03-10 08:11:22", + "window_end": "2026-03-10 08:18:05", + "usernames": ["admin", "deploy", "guest", "root", "test"], + "summary": "203.0.113.10 targeted 5 usernames within 15 minutes." + }, + { + "rule": "sudo_burst", + "subject_kind": "username", + "subject": "alice", + "event_count": 3, + "window_start": "2026-03-10 08:21:00", + "window_end": "2026-03-10 08:24:15", + "usernames": [], + "summary": "alice ran 3 sudo commands within 5 minutes." + } + ], + "warnings": [ + {"line_number": 15, "reason": "unrecognized auth pattern: sshd_connection_closed_preauth"}, + {"line_number": 16, "reason": "unrecognized auth pattern: sshd_timeout_or_disconnection"} + ] +} diff --git a/tests/fixtures/report_contracts/journalctl_short_full/report.md b/tests/fixtures/report_contracts/journalctl_short_full/report.md new file mode 100644 index 0000000..a122bd9 --- /dev/null +++ b/tests/fixtures/report_contracts/journalctl_short_full/report.md @@ -0,0 +1,48 @@ +# LogLens Report + +## Summary + +- Input: `tests/fixtures/report_contracts/journalctl_short_full/input.log` +- Input mode: journalctl_short_full +- Timezone present: true +- Total lines: 16 +- Parsed lines: 14 +- Unparsed lines: 2 +- Parse success rate: 87.50% +- Parsed events: 14 +- Findings: 3 +- Parser warnings: 2 + +## Findings + +| Rule | Subject | Count | Window | Notes | +| --- | --- | ---: | --- | --- | +| brute_force | 203.0.113.10 | 5 | 2026-03-10 08:11:22 -> 2026-03-10 08:18:05 | 5 failed SSH attempts from 203.0.113.10 within 10 minutes. | +| multi_user_probing | 203.0.113.10 | 5 | 2026-03-10 08:11:22 -> 2026-03-10 08:18:05 | 203.0.113.10 targeted 5 usernames within 15 minutes. Usernames: admin, deploy, guest, root, test | +| sudo_burst | alice | 3 | 2026-03-10 08:21:00 -> 2026-03-10 08:24:15 | alice ran 3 sudo commands within 5 minutes. | + +## Event Counts + +| Event Type | Count | +| --- | ---: | +| ssh_failed_password | 4 | +| ssh_accepted_password | 1 | +| ssh_invalid_user | 3 | +| ssh_failed_publickey | 1 | +| pam_auth_failure | 1 | +| session_opened | 1 | +| sudo_command | 3 | + +## Parser Quality + +| Unknown Pattern | Count | +| --- | ---: | +| sshd_connection_closed_preauth | 1 | +| sshd_timeout_or_disconnection | 1 | + +## Parser Warnings + +| Line | Reason | +| ---: | --- | +| 15 | unrecognized auth pattern: sshd_connection_closed_preauth | +| 16 | unrecognized auth pattern: sshd_timeout_or_disconnection | diff --git a/tests/fixtures/report_contracts/syslog_legacy/input.log b/tests/fixtures/report_contracts/syslog_legacy/input.log new file mode 100644 index 0000000..57763f5 --- /dev/null +++ b/tests/fixtures/report_contracts/syslog_legacy/input.log @@ -0,0 +1,16 @@ +Mar 10 08:11:22 example-host sshd[1234]: Failed password for invalid user admin from 203.0.113.10 port 51022 ssh2 +Mar 10 08:12:05 example-host sshd[1235]: Failed password for root from 203.0.113.10 port 51030 ssh2 +Mar 10 08:13:10 example-host sshd[1236]: Failed password for test from 203.0.113.10 port 51040 ssh2 +Mar 10 08:14:44 example-host sshd[1237]: Failed password for guest from 203.0.113.10 port 51050 ssh2 +Mar 10 08:18:05 example-host sshd[1238]: Failed password for invalid user deploy from 203.0.113.10 port 51060 ssh2 +Mar 10 08:20:10 example-host sshd[1240]: Accepted password for alice from 203.0.113.20 port 51111 ssh2 +Mar 10 08:21:00 example-host sudo: alice : TTY=pts/0 ; PWD=/home/alice ; USER=root ; COMMAND=/usr/bin/systemctl restart ssh +Mar 10 08:22:10 example-host sudo: alice : TTY=pts/0 ; PWD=/home/alice ; USER=root ; COMMAND=/usr/bin/journalctl -xe +Mar 10 08:24:15 example-host sudo: alice : TTY=pts/0 ; PWD=/home/alice ; USER=root ; COMMAND=/usr/bin/vi /etc/ssh/sshd_config +Mar 10 08:25:30 example-host sshd[1241]: Failed password for bob from 203.0.113.30 port 51234 ssh2 +Mar 10 08:26:02 example-host sshd[1242]: Invalid user backup from 203.0.113.31 port 51236 +Mar 10 08:27:10 example-host sshd[1243]: Failed publickey for invalid user svc-backup from 203.0.113.40 port 51240 ssh2 +Mar 10 08:28:33 example-host pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=203.0.113.41 user=alice +Mar 10 08:29:50 example-host pam_unix(sudo:session): session opened for user root by alice(uid=0) +Mar 10 08:30:12 example-host sshd[1244]: Connection closed by authenticating user alice 203.0.113.50 port 51290 [preauth] +Mar 10 08:31:18 example-host sshd[1245]: Timeout, client not responding from 203.0.113.51 port 51291 diff --git a/tests/fixtures/report_contracts/syslog_legacy/report.json b/tests/fixtures/report_contracts/syslog_legacy/report.json new file mode 100644 index 0000000..c80f7c4 --- /dev/null +++ b/tests/fixtures/report_contracts/syslog_legacy/report.json @@ -0,0 +1,65 @@ +{ + "tool": "LogLens", + "input": "tests/fixtures/report_contracts/syslog_legacy/input.log", + "input_mode": "syslog_legacy", + "assume_year": 2026, + "timezone_present": false, + "parser_quality": { + "total_lines": 16, + "parsed_lines": 14, + "unparsed_lines": 2, + "parse_success_rate": 0.8750, + "top_unknown_patterns": [ + {"pattern": "sshd_connection_closed_preauth", "count": 1}, + {"pattern": "sshd_timeout_or_disconnection", "count": 1} + ] + }, + "parsed_event_count": 14, + "warning_count": 2, + "finding_count": 3, + "event_counts": [ + {"event_type": "ssh_failed_password", "count": 4}, + {"event_type": "ssh_accepted_password", "count": 1}, + {"event_type": "ssh_invalid_user", "count": 3}, + {"event_type": "ssh_failed_publickey", "count": 1}, + {"event_type": "pam_auth_failure", "count": 1}, + {"event_type": "session_opened", "count": 1}, + {"event_type": "sudo_command", "count": 3} + ], + "findings": [ + { + "rule": "brute_force", + "subject_kind": "source_ip", + "subject": "203.0.113.10", + "event_count": 5, + "window_start": "2026-03-10 08:11:22", + "window_end": "2026-03-10 08:18:05", + "usernames": [], + "summary": "5 failed SSH attempts from 203.0.113.10 within 10 minutes." + }, + { + "rule": "multi_user_probing", + "subject_kind": "source_ip", + "subject": "203.0.113.10", + "event_count": 5, + "window_start": "2026-03-10 08:11:22", + "window_end": "2026-03-10 08:18:05", + "usernames": ["admin", "deploy", "guest", "root", "test"], + "summary": "203.0.113.10 targeted 5 usernames within 15 minutes." + }, + { + "rule": "sudo_burst", + "subject_kind": "username", + "subject": "alice", + "event_count": 3, + "window_start": "2026-03-10 08:21:00", + "window_end": "2026-03-10 08:24:15", + "usernames": [], + "summary": "alice ran 3 sudo commands within 5 minutes." + } + ], + "warnings": [ + {"line_number": 15, "reason": "unrecognized auth pattern: sshd_connection_closed_preauth"}, + {"line_number": 16, "reason": "unrecognized auth pattern: sshd_timeout_or_disconnection"} + ] +} diff --git a/tests/fixtures/report_contracts/syslog_legacy/report.md b/tests/fixtures/report_contracts/syslog_legacy/report.md new file mode 100644 index 0000000..b26d176 --- /dev/null +++ b/tests/fixtures/report_contracts/syslog_legacy/report.md @@ -0,0 +1,49 @@ +# LogLens Report + +## Summary + +- Input: `tests/fixtures/report_contracts/syslog_legacy/input.log` +- Input mode: syslog_legacy +- Assume year: 2026 +- Timezone present: false +- Total lines: 16 +- Parsed lines: 14 +- Unparsed lines: 2 +- Parse success rate: 87.50% +- Parsed events: 14 +- Findings: 3 +- Parser warnings: 2 + +## Findings + +| Rule | Subject | Count | Window | Notes | +| --- | --- | ---: | --- | --- | +| brute_force | 203.0.113.10 | 5 | 2026-03-10 08:11:22 -> 2026-03-10 08:18:05 | 5 failed SSH attempts from 203.0.113.10 within 10 minutes. | +| multi_user_probing | 203.0.113.10 | 5 | 2026-03-10 08:11:22 -> 2026-03-10 08:18:05 | 203.0.113.10 targeted 5 usernames within 15 minutes. Usernames: admin, deploy, guest, root, test | +| sudo_burst | alice | 3 | 2026-03-10 08:21:00 -> 2026-03-10 08:24:15 | alice ran 3 sudo commands within 5 minutes. | + +## Event Counts + +| Event Type | Count | +| --- | ---: | +| ssh_failed_password | 4 | +| ssh_accepted_password | 1 | +| ssh_invalid_user | 3 | +| ssh_failed_publickey | 1 | +| pam_auth_failure | 1 | +| session_opened | 1 | +| sudo_command | 3 | + +## Parser Quality + +| Unknown Pattern | Count | +| --- | ---: | +| sshd_connection_closed_preauth | 1 | +| sshd_timeout_or_disconnection | 1 | + +## Parser Warnings + +| Line | Reason | +| ---: | --- | +| 15 | unrecognized auth pattern: sshd_connection_closed_preauth | +| 16 | unrecognized auth pattern: sshd_timeout_or_disconnection | diff --git a/tests/test_report_contracts.cpp b/tests/test_report_contracts.cpp new file mode 100644 index 0000000..e4985e3 --- /dev/null +++ b/tests/test_report_contracts.cpp @@ -0,0 +1,271 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void expect(bool condition, const std::string& message) { + if (!condition) { + throw std::runtime_error(message); + } +} + +std::filesystem::path repo_root() { + const std::filesystem::path source_path{__FILE__}; + std::vector candidates; + + if (source_path.is_absolute()) { + candidates.push_back(source_path); + } else { + const auto cwd = std::filesystem::current_path(); + candidates.push_back(cwd / source_path); + candidates.push_back(cwd.parent_path() / source_path); + } + + for (const auto& candidate : candidates) { + if (std::filesystem::exists(candidate)) { + return candidate.parent_path().parent_path(); + } + } + + throw std::runtime_error("unable to resolve repository root from test source path"); +} + +std::string read_file(const std::filesystem::path& path) { + std::ifstream input(path); + if (!input) { + throw std::runtime_error("unable to read file: " + path.string()); + } + + return std::string((std::istreambuf_iterator(input)), std::istreambuf_iterator()); +} + +std::string normalize_line_endings(std::string value) { + value.erase(std::remove(value.begin(), value.end(), '\r'), value.end()); + return value; +} + +std::vector split_lines(const std::string& content) { + std::vector lines; + std::string current; + + for (const char ch : normalize_line_endings(content)) { + if (ch == '\n') { + lines.push_back(current); + current.clear(); + } else { + current += ch; + } + } + + if (!current.empty()) { + lines.push_back(current); + } + + return lines; +} + +std::string trim(std::string_view value) { + std::size_t start = 0; + while (start < value.size() && (value[start] == ' ' || value[start] == '\t')) { + ++start; + } + + std::size_t end = value.size(); + while (end > start && (value[end - 1] == ' ' || value[end - 1] == '\t')) { + --end; + } + + return std::string(value.substr(start, end - start)); +} + +bool starts_with(std::string_view value, std::string_view prefix) { + return value.size() >= prefix.size() && value.substr(0, prefix.size()) == prefix; +} + +bool is_markdown_separator_row(std::string_view line) { + return starts_with(line, "| ---"); +} + +std::vector extract_markdown_contract_lines(const std::string& markdown) { + std::vector contract_lines; + + for (const auto& raw_line : split_lines(markdown)) { + const auto line = trim(raw_line); + if (line.empty() || is_markdown_separator_row(line)) { + continue; + } + + if (line == "# LogLens Report" + || starts_with(line, "## ") + || starts_with(line, "- Input: ") + || starts_with(line, "- Input mode: ") + || starts_with(line, "- Assume year: ") + || starts_with(line, "- Timezone present: ") + || starts_with(line, "- Total lines: ") + || starts_with(line, "- Parsed lines: ") + || starts_with(line, "- Unparsed lines: ") + || starts_with(line, "- Parse success rate: ") + || starts_with(line, "- Parsed events: ") + || starts_with(line, "- Findings: ") + || starts_with(line, "- Parser warnings: ") + || starts_with(line, "| ") + || starts_with(line, "No configured detections matched") + || starts_with(line, "All analyzed lines matched") + || starts_with(line, "No malformed lines were skipped")) { + contract_lines.push_back(line); + } + } + + return contract_lines; +} + +std::vector extract_json_contract_lines(const std::string& json) { + std::vector contract_lines; + + for (const auto& raw_line : split_lines(json)) { + const auto line = trim(raw_line); + if (line.empty()) { + continue; + } + + if (starts_with(line, "\"tool\": ") + || starts_with(line, "\"input\": ") + || starts_with(line, "\"input_mode\": ") + || starts_with(line, "\"assume_year\": ") + || starts_with(line, "\"timezone_present\": ") + || starts_with(line, "\"total_lines\": ") + || starts_with(line, "\"parsed_lines\": ") + || starts_with(line, "\"unparsed_lines\": ") + || starts_with(line, "\"parse_success_rate\": ") + || starts_with(line, "\"parsed_event_count\": ") + || starts_with(line, "\"warning_count\": ") + || starts_with(line, "\"finding_count\": ") + || starts_with(line, "{\"pattern\": ") + || starts_with(line, "{\"event_type\": ") + || starts_with(line, "\"rule\": ") + || starts_with(line, "\"subject_kind\": ") + || starts_with(line, "\"subject\": ") + || starts_with(line, "\"event_count\": ") + || starts_with(line, "\"window_start\": ") + || starts_with(line, "\"window_end\": ") + || starts_with(line, "\"usernames\": ") + || starts_with(line, "\"summary\": ") + || starts_with(line, "{\"line_number\": ")) { + contract_lines.push_back(line); + } + } + + return contract_lines; +} + +std::string quote_argument(std::string_view value) { + return "\"" + std::string(value) + "\""; +} + +std::string build_command(const std::string& invocation) { +#ifdef _WIN32 + return "cmd /c \"" + invocation + "\""; +#else + return invocation; +#endif +} + +void expect_equal_lines(const std::vector& actual, + const std::vector& expected, + const std::string& message) { + if (actual == expected) { + return; + } + + std::string details = message + "\nexpected:\n"; + for (const auto& line : expected) { + details += " " + line + '\n'; + } + details += "actual:\n"; + for (const auto& line : actual) { + details += " " + line + '\n'; + } + + throw std::runtime_error(details); +} + +void run_report_contract_case(const std::filesystem::path& loglens_exe, + const std::filesystem::path& fixture_directory, + const std::filesystem::path& output_root, + const std::string& mode_argument, + const std::string& extra_arguments = {}) { + const auto repo = repo_root(); + const auto relative_input = std::filesystem::relative(fixture_directory / "input.log", repo).generic_string(); + const auto case_output = output_root / fixture_directory.filename(); + + std::filesystem::remove_all(case_output); + std::filesystem::create_directories(case_output); + + std::string invocation = quote_argument(loglens_exe.generic_string()) + + " --mode " + mode_argument; + if (!extra_arguments.empty()) { + invocation += " " + extra_arguments; + } + invocation += " " + quote_argument(relative_input) + + " " + quote_argument(case_output.generic_string()); + + const int exit_code = std::system(build_command(invocation).c_str()); + expect(exit_code == 0, "expected report contract CLI run to succeed for " + fixture_directory.filename().string()); + + const auto actual_markdown = read_file(case_output / "report.md"); + const auto actual_json = read_file(case_output / "report.json"); + const auto golden_markdown = read_file(fixture_directory / "report.md"); + const auto golden_json = read_file(fixture_directory / "report.json"); + + expect_equal_lines( + extract_markdown_contract_lines(actual_markdown), + extract_markdown_contract_lines(golden_markdown), + "markdown contract mismatch for " + fixture_directory.filename().string()); + expect_equal_lines( + extract_json_contract_lines(actual_json), + extract_json_contract_lines(golden_json), + "json contract mismatch for " + fixture_directory.filename().string()); +} + +} // namespace + +int main(int argc, char* argv[]) { + if (argc != 3) { + throw std::runtime_error("expected arguments: "); + } + + const auto original_cwd = std::filesystem::current_path(); + const auto repo = repo_root(); + std::filesystem::current_path(repo); + + try { + const std::filesystem::path loglens_exe = std::filesystem::absolute(argv[1]); + const std::filesystem::path output_root = std::filesystem::absolute(argv[2]); + const auto fixture_root = repo / "tests" / "fixtures" / "report_contracts"; + + run_report_contract_case( + loglens_exe, + fixture_root / "syslog_legacy", + output_root, + "syslog", + "--year 2026"); + run_report_contract_case( + loglens_exe, + fixture_root / "journalctl_short_full", + output_root, + "journalctl-short-full"); + } catch (...) { + std::filesystem::current_path(original_cwd); + throw; + } + + std::filesystem::current_path(original_cwd); + return 0; +}