From 8f00c0aa4e84ae7c7204e5bea238122d81650aab Mon Sep 17 00:00:00 2001 From: Vitalii Parfonov Date: Tue, 1 Sep 2026 21:27:34 +0300 Subject: [PATCH] fix(detect_exceptions): use targeted patterns instead of catch-all for Java continuations Replace the generic catch-all rule (^.+$) with targeted continuation patterns that handle non-indented lines between exception headers and stack traces (LOG-9995) without greedily consuming independent JSON log records (LOG-9963). The catch-all caused detectMultilineException to merge unrelated single-line JSON logs into one event whenever a message contained an exception keyword like "Exception:" or "Error:". Targeted patterns added: - ^[A-Z]+-?\d+ error codes (ORA-12521, TNS-xxxxx) - ^[\t ]*\( parenthesized lines (CONNECTION_ID=...) - ^[\t ]+\S indented non-stack-frame lines Ref: LOG-9963, LOG-9995 Co-Authored-By: Claude Opus 4.6 Signed-off-by: Vitalii Parfonov --- Makefile | 2 +- .../detect_exceptions/exception_detector.rs | 126 +++++++-- src/transforms/detect_exceptions/mod.rs | 265 ++++++++++++++++++ src/transforms/detect_exceptions/rules.rs | 11 + 4 files changed, 372 insertions(+), 32 deletions(-) diff --git a/Makefile b/Makefile index a5ec4ac901203..cd65c79a7699f 100644 --- a/Makefile +++ b/Makefile @@ -752,7 +752,7 @@ clippy-fix: .PHONY: fmt fmt: - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) fmt + cargo fmt --all .PHONY: build-licenses build-licenses: diff --git a/src/transforms/detect_exceptions/exception_detector.rs b/src/transforms/detect_exceptions/exception_detector.rs index 68b0321b44d72..224703557f9dd 100644 --- a/src/transforms/detect_exceptions/exception_detector.rs +++ b/src/transforms/detect_exceptions/exception_detector.rs @@ -292,6 +292,7 @@ mod exception_detector_tests { check_exception(java_simple_exception(), false); check_exception(java_complex_exception(), false); check_exception(java_nested_exception(), false); + check_exception(java_exception_with_non_indented_continuation(), false); } const fn java_simple_exception() -> &'static str { @@ -366,6 +367,18 @@ com.sun.mail.smtp.SMTPAddressFailedException: 550 5.7.1 <[REDACTED_EMAIL_ADDRESS at com.nethunt.crm.api.server.adminsync.AutomaticEmailFacade.sendWithSmtp(AutomaticEmailFacade.java:229) ... 12 more Caused by: com.sun.mail.smtp.SMTPAddressFailedException: 550 5.7.1 <[REDACTED_EMAIL_ADDRESS]>... Relaying denied + at com.sun.mail.smtp.SMTPTransport.rcptTo(SMTPTransport.java:2064) + ... 12 more + " + } + + const fn java_exception_with_non_indented_continuation() -> &'static str { + " +java.sql.SQLException: Listener refused the connection with the following error: +ORA-12521, TNS:listener does not currently know of instance requested in connect descriptor + (CONNECTION_ID=r6n2ZPL0TqS/BLDhIydj+A==) + at oracle.jdbc.driver.T4CConnection.handleLogonNetException(T4CConnection.java:893) + at oracle.jdbc.driver.T4CConnection.logon(T4CConnection.java:698) " } @@ -563,37 +576,33 @@ created by net/http.(*Server).Serve " } - fn rails_exception() -> &'static str { - // The blank line between the error header and the stack trace must contain - // exactly two spaces — the ruby rule `^ $` matches it to transition from - // RubyBeforeRailsTrace to Ruby. Using concat! so rustfmt cannot strip them. - concat!( - "\n", - " ActionController::RoutingError (No route matches [GET] \"/settings\"):\n", - " \n", - " actionpack (5.1.4) lib/action_dispatch/middleware/debug_exceptions.rb:63:in `call'\n", - " actionpack (5.1.4) lib/action_dispatch/middleware/show_exceptions.rb:31:in `call'\n", - " railties (5.1.4) lib/rails/rack/logger.rb:36:in `call_app'\n", - " railties (5.1.4) lib/rails/rack/logger.rb:24:in `block in call'\n", - " activesupport (5.1.4) lib/active_support/tagged_logging.rb:69:in `block in tagged'\n", - " activesupport (5.1.4) lib/active_support/tagged_logging.rb:26:in `tagged'\n", - " activesupport (5.1.4) lib/active_support/tagged_logging.rb:69:in `tagged'\n", - " railties (5.1.4) lib/rails/rack/logger.rb:24:in `call'\n", - " actionpack (5.1.4) lib/action_dispatch/middleware/remote_ip.rb:79:in `call'\n", - " actionpack (5.1.4) lib/action_dispatch/middleware/request_id.rb:25:in `call'\n", - " rack (2.0.3) lib/rack/method_override.rb:22:in `call'\n", - " rack (2.0.3) lib/rack/runtime.rb:22:in `call'\n", - " activesupport (5.1.4) lib/active_support/cache/strategy/local_cache_middleware.rb:27:in `call'\n", - " actionpack (5.1.4) lib/action_dispatch/middleware/executor.rb:12:in `call'\n", - " rack (2.0.3) lib/rack/sendfile.rb:111:in `call'\n", - " railties (5.1.4) lib/rails/engine.rb:522:in `call'\n", - " puma (3.10.0) lib/puma/configuration.rb:225:in `call'\n", - " puma (3.10.0) lib/puma/server.rb:605:in `handle_request'\n", - " puma (3.10.0) lib/puma/server.rb:437:in `process_client'\n", - " puma (3.10.0) lib/puma/server.rb:301:in `block in run'\n", - " puma (3.10.0) lib/puma/thread_pool.rb:120:in `block in spawn_thread'\n", - " ", - ) + #[rustfmt::skip] + const fn rails_exception() -> &'static str { + r#" + ActionController::RoutingError (No route matches [GET] "/settings"): + + actionpack (5.1.4) lib/action_dispatch/middleware/debug_exceptions.rb:63:in `call' + actionpack (5.1.4) lib/action_dispatch/middleware/show_exceptions.rb:31:in `call' + railties (5.1.4) lib/rails/rack/logger.rb:36:in `call_app' + railties (5.1.4) lib/rails/rack/logger.rb:24:in `block in call' + activesupport (5.1.4) lib/active_support/tagged_logging.rb:69:in `block in tagged' + activesupport (5.1.4) lib/active_support/tagged_logging.rb:26:in `tagged' + activesupport (5.1.4) lib/active_support/tagged_logging.rb:69:in `tagged' + railties (5.1.4) lib/rails/rack/logger.rb:24:in `call' + actionpack (5.1.4) lib/action_dispatch/middleware/remote_ip.rb:79:in `call' + actionpack (5.1.4) lib/action_dispatch/middleware/request_id.rb:25:in `call' + rack (2.0.3) lib/rack/method_override.rb:22:in `call' + rack (2.0.3) lib/rack/runtime.rb:22:in `call' + activesupport (5.1.4) lib/active_support/cache/strategy/local_cache_middleware.rb:27:in `call' + actionpack (5.1.4) lib/action_dispatch/middleware/executor.rb:12:in `call' + rack (2.0.3) lib/rack/sendfile.rb:111:in `call' + railties (5.1.4) lib/rails/engine.rb:522:in `call' + puma (3.10.0) lib/puma/configuration.rb:225:in `call' + puma (3.10.0) lib/puma/server.rb:605:in `handle_request' + puma (3.10.0) lib/puma/server.rb:437:in `process_client' + puma (3.10.0) lib/puma/server.rb:301:in `block in run' + puma (3.10.0) lib/puma/thread_pool.rb:120:in `block in spawn_thread' + "# } #[test] @@ -614,6 +623,61 @@ Exception: ('spam', 'eggs') "# } + #[test] + fn test_exception_keyword_resets_on_non_matching_line() { + // LOG-9963: After an exception keyword triggers JavaAfterException, + // a line that doesn't match any continuation pattern must reset + // the state machine to StartState (NoTrace or EndTrace), NOT + // continue accumulating. + let mut detector = ExceptionDetector { + state_machine: Arc::new(get_state_machines(default_programming_languages())), + current_state: ExceptionState::StartState, + }; + + // Exception keyword triggers the state machine. + let status = detector.update(&"SecretConfigException: config error".to_string()); + assert_eq!(InsideTrace, status); + assert_ne!(ExceptionState::StartState, detector.current_state); + + // Non-matching line: transition from JavaAfterException fails, retry + // from StartState also fails → NoTrace. The TraceAccumulator treats + // this as "flush accumulated + emit current", so no merging occurs. + let status = detector.update(&"Normal operation completed".to_string()); + assert_eq!(NoTrace, status); + assert_eq!(ExceptionState::StartState, detector.current_state); + + // Subsequent non-exception line: already in StartState, no match. + let status = detector.update(&"Another normal log".to_string()); + assert_eq!(NoTrace, status); + assert_eq!(ExceptionState::StartState, detector.current_state); + } + + #[test] + fn test_consecutive_exception_keywords_each_start_new_trace() { + // LOG-9963: Independent lines each containing exception keywords + // must each be treated as separate trace starts, not merged. + let mut detector = ExceptionDetector { + state_machine: Arc::new(get_state_machines(default_programming_languages())), + current_state: ExceptionState::StartState, + }; + + let status = detector.update(&"ConnectTimeoutException: host unreachable".to_string()); + assert_eq!(InsideTrace, status); + + // Second exception keyword: should end previous trace and start new. + let status = detector.update(&"RestException: upstream returned 503".to_string()); + assert_eq!(StartTrace, status); + + // Third exception keyword: same behavior. + let status = detector.update(&"IllegalArgumentException: null".to_string()); + assert_eq!(StartTrace, status); + + // Non-exception line: no match from JavaAfterException or StartState. + let status = detector.update(&"Request processed".to_string()); + assert_eq!(NoTrace, status); + assert_eq!(ExceptionState::StartState, detector.current_state); + } + fn split(line: &str) -> Vec<&str> { line.trim().split("\n").collect::>() } diff --git a/src/transforms/detect_exceptions/mod.rs b/src/transforms/detect_exceptions/mod.rs index bcd62e2033e91..bd936623bb515 100644 --- a/src/transforms/detect_exceptions/mod.rs +++ b/src/transforms/detect_exceptions/mod.rs @@ -393,4 +393,269 @@ Jul 09, 2015 3:23:29 PM com.google.devtools.search.cloud.feeder.MakeLog: Runtime assert_eq!(output_2["message"], java_simple_log.trim().into()); assert_eq!(output_2["counter"], Value::from(6)); } + + #[tokio::test] + async fn test_exception_with_non_indented_continuation_lines() { + // LOG-9995: Oracle ORA errors have non-indented lines between the + // exception header and the stack trace. Targeted continuation patterns + // must group these correctly. + let detect_exceptions = toml::from_str::( + r#" +languages = ["Java"] +"#, + ) + .unwrap() + .build(&TransformContext::default()) + .await + .unwrap(); + + let detect_exceptions = detect_exceptions.into_task(); + + let exception_with_continuation = "\ +java.sql.SQLException: Listener refused the connection with the following error: +ORA-12521, TNS:listener does not currently know of instance requested in connect descriptor + (CONNECTION_ID=r6n2ZPL0TqS/BLDhIydj+A==) + at oracle.jdbc.driver.T4CConnection.handleLogonNetException(T4CConnection.java:893) + at oracle.jdbc.driver.T4CConnection.logon(T4CConnection.java:698)"; + let regular_log = "2026-06-02 13:55:59.506 INFO normal log message"; + + let lines = format!("{}\n{}", exception_with_continuation, regular_log); + + let mut counter = 0; + let input_events: Vec = lines + .split("\n") + .map(|line| { + let mut le = LogEvent::from(line); + le.insert("counter", counter); + counter += 1; + Event::Log(le) + }) + .collect(); + + let in_stream = Box::pin(stream::iter(input_events)); + let mut out_stream = detect_exceptions.transform_events(in_stream); + + let output_1 = out_stream.next().await.unwrap().into_log(); + assert_eq!( + output_1["message"], + exception_with_continuation.into(), + "All exception lines including non-indented continuations must be merged" + ); + assert_eq!(output_1["counter"], Value::from(0)); + + let output_2 = out_stream.next().await.unwrap().into_log(); + assert_eq!(output_2["message"], regular_log.into()); + assert_eq!(output_2["counter"], Value::from(5)); + } + + #[tokio::test] + async fn test_json_messages_with_exception_keywords_not_merged() { + // LOG-9963: When parse_json runs before detect_exceptions, the message + // field contains parsed string values. If a message contains an + // exception keyword, subsequent independent messages must NOT be merged. + let detect_exceptions = toml::from_str::( + r#" +languages = ["All"] +"#, + ) + .unwrap() + .build(&TransformContext::default()) + .await + .unwrap(); + + let detect_exceptions = detect_exceptions.into_task(); + + let msg1 = "SecretConfigException: Failed to load configuration from vault"; + let msg2 = "Processing request for user 12345"; + let msg3 = "Response sent in 42ms"; + + let mut counter = 0; + let input_events: Vec = [msg1, msg2, msg3] + .iter() + .map(|line| { + let mut le = LogEvent::from(*line); + le.insert("counter", counter); + counter += 1; + Event::Log(le) + }) + .collect(); + + let in_stream = Box::pin(stream::iter(input_events)); + let mut out_stream = detect_exceptions.transform_events(in_stream); + + let output_1 = out_stream.next().await.unwrap().into_log(); + assert_eq!(output_1["message"], msg1.into()); + assert_eq!(output_1["counter"], Value::from(0)); + + let output_2 = out_stream.next().await.unwrap().into_log(); + assert_eq!(output_2["message"], msg2.into()); + assert_eq!(output_2["counter"], Value::from(1)); + + let output_3 = out_stream.next().await.unwrap().into_log(); + assert_eq!(output_3["message"], msg3.into()); + assert_eq!(output_3["counter"], Value::from(2)); + } + + #[tokio::test] + async fn test_raw_json_log_lines_not_merged() { + // LOG-9963: When detect_exceptions runs WITHOUT parse_json, the message + // field contains the entire raw JSON line. Exception keywords embedded + // inside JSON field values must not trigger merging of subsequent + // independent JSON records. + let detect_exceptions = toml::from_str::( + r#" +languages = ["All"] +"#, + ) + .unwrap() + .build(&TransformContext::default()) + .await + .unwrap(); + + let detect_exceptions = detect_exceptions.into_task(); + + let json1 = r#"{"timestamp":"2026-08-28T10:00:00Z","level":"ERROR","message":"SecretConfigException: Failed to load config","stackTrace":"java.lang.Throwable\n\tat com.example.Config.load(Config.java:42)"}"#; + let json2 = r#"{"timestamp":"2026-08-28T10:00:01Z","level":"INFO","message":"Processing request for user 12345"}"#; + let json3 = r#"{"timestamp":"2026-08-28T10:00:02Z","level":"INFO","message":"Response sent in 42ms"}"#; + + let mut counter = 0; + let input_events: Vec = [json1, json2, json3] + .iter() + .map(|line| { + let mut le = LogEvent::from(*line); + le.insert("counter", counter); + counter += 1; + Event::Log(le) + }) + .collect(); + + let in_stream = Box::pin(stream::iter(input_events)); + let mut out_stream = detect_exceptions.transform_events(in_stream); + + let output_1 = out_stream.next().await.unwrap().into_log(); + assert_eq!(output_1["message"], json1.into()); + assert_eq!(output_1["counter"], Value::from(0)); + + let output_2 = out_stream.next().await.unwrap().into_log(); + assert_eq!(output_2["message"], json2.into()); + assert_eq!(output_2["counter"], Value::from(1)); + + let output_3 = out_stream.next().await.unwrap().into_log(); + assert_eq!(output_3["message"], json3.into()); + assert_eq!(output_3["counter"], Value::from(2)); + } + + #[tokio::test] + async fn test_consecutive_exception_keyword_messages_not_merged() { + // LOG-9963: Multiple independent messages each containing exception + // keywords must be emitted as separate events, not merged. + let detect_exceptions = toml::from_str::( + r#" +languages = ["All"] +"#, + ) + .unwrap() + .build(&TransformContext::default()) + .await + .unwrap(); + + let detect_exceptions = detect_exceptions.into_task(); + + let msg1 = "ConnectTimeoutException: connection to host timed out"; + let msg2 = "RestException: upstream returned 503"; + let msg3 = "IllegalArgumentException: parameter must not be null"; + let msg4 = "Normal log message without exception keywords"; + + let mut counter = 0; + let input_events: Vec = [msg1, msg2, msg3, msg4] + .iter() + .map(|line| { + let mut le = LogEvent::from(*line); + le.insert("counter", counter); + counter += 1; + Event::Log(le) + }) + .collect(); + + let in_stream = Box::pin(stream::iter(input_events)); + let mut out_stream = detect_exceptions.transform_events(in_stream); + + let output_1 = out_stream.next().await.unwrap().into_log(); + assert_eq!(output_1["message"], msg1.into()); + assert_eq!(output_1["counter"], Value::from(0)); + + let output_2 = out_stream.next().await.unwrap().into_log(); + assert_eq!(output_2["message"], msg2.into()); + assert_eq!(output_2["counter"], Value::from(1)); + + let output_3 = out_stream.next().await.unwrap().into_log(); + assert_eq!(output_3["message"], msg3.into()); + assert_eq!(output_3["counter"], Value::from(2)); + + let output_4 = out_stream.next().await.unwrap().into_log(); + assert_eq!(output_4["message"], msg4.into()); + assert_eq!(output_4["counter"], Value::from(3)); + } + + #[tokio::test] + async fn test_real_exception_still_grouped_after_json_fix() { + // Verify that removing the catch-all does not break grouping of + // real multi-line exceptions (standard indented stack traces). + let detect_exceptions = toml::from_str::( + r#" +languages = ["Java"] +"#, + ) + .unwrap() + .build(&TransformContext::default()) + .await + .unwrap(); + + let detect_exceptions = detect_exceptions.into_task(); + + let exception = "\ +java.lang.NullPointerException: null + at com.example.MyClass.myMethod(MyClass.java:42) + at com.example.Main.main(Main.java:10)"; + let normal_log_1 = "Application started successfully"; + let exception_keyword_log = "JobException: scheduled task failed"; + let normal_log_2 = "Shutting down gracefully"; + + let lines = format!( + "{}\n{}\n{}\n{}", + exception, normal_log_1, exception_keyword_log, normal_log_2 + ); + + let mut counter = 0; + let input_events: Vec = lines + .split("\n") + .map(|line| { + let mut le = LogEvent::from(line); + le.insert("counter", counter); + counter += 1; + Event::Log(le) + }) + .collect(); + + let in_stream = Box::pin(stream::iter(input_events)); + let mut out_stream = detect_exceptions.transform_events(in_stream); + + // The real exception (3 lines) must still be grouped. + let output_1 = out_stream.next().await.unwrap().into_log(); + assert_eq!(output_1["message"], exception.into()); + assert_eq!(output_1["counter"], Value::from(0)); + + // Each subsequent line is its own event. + let output_2 = out_stream.next().await.unwrap().into_log(); + assert_eq!(output_2["message"], normal_log_1.into()); + assert_eq!(output_2["counter"], Value::from(3)); + + let output_3 = out_stream.next().await.unwrap().into_log(); + assert_eq!(output_3["message"], exception_keyword_log.into()); + assert_eq!(output_3["counter"], Value::from(4)); + + let output_4 = out_stream.next().await.unwrap().into_log(); + assert_eq!(output_4["message"], normal_log_2.into()); + assert_eq!(output_4["counter"], Value::from(5)); + } } diff --git a/src/transforms/detect_exceptions/rules.rs b/src/transforms/detect_exceptions/rules.rs index 69f8631483ebd..1d23ef532ac6a 100644 --- a/src/transforms/detect_exceptions/rules.rs +++ b/src/transforms/detect_exceptions/rules.rs @@ -106,6 +106,17 @@ fn java_rules() -> Vec> { r"^[\t ]*... \d+ (?:more|common frames omitted)", Java, ), + // Targeted continuation patterns for non-indented lines between the + // exception header and stack trace (e.g. Oracle ORA errors, LOG-9565). + // These replace a former generic catch-all (^.+$) that was reverted + // because it greedily consumed independent JSON log records (LOG-9963). + rule( + vec![JavaAfterException], + r"^[A-Z]+-?\d+", + JavaAfterException, + ), + rule(vec![JavaAfterException], r"^[\t ]*\(", JavaAfterException), + rule(vec![JavaAfterException], r"^[\t ]+\S", JavaAfterException), ] }