From 7af8809ca36e314baa01f86f11bb3a2a24d246c6 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Thu, 17 Sep 2026 08:34:04 -0700 Subject: [PATCH 01/19] Add allMatches to Traverse function to improve performance --- .../DeterministicFsaTraversalMethod.cs | 40 ++++++++++++++++- .../DeterministicFstTraversalMethod.cs | 3 +- src/SIL.Machine/FiniteState/Fst.cs | 2 +- .../FiniteState/ITraversalMethod.cs | 3 +- .../NondeterministicFsaTraversalMethod.cs | 43 ++++++++++++++++++- .../NondeterministicFstTraversalMethod.cs | 3 +- .../FiniteState/TraversalMethodBase.cs | 3 +- 7 files changed, 89 insertions(+), 8 deletions(-) diff --git a/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs index 5470fa689..1be7676ab 100644 --- a/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs @@ -1,6 +1,8 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using SIL.Machine.Annotations; using SIL.Machine.FeatureModel; +using SIL.ObjectModel; namespace SIL.Machine.FiniteState { @@ -22,7 +24,8 @@ public override IEnumerable> Traverse( ref int annIndex, Register[,] initRegisters, IList initCmds, - ISet initAnns + ISet initAnns, + bool allMatches ) { Stack> instStack = InitializeStack( @@ -33,6 +36,13 @@ ISet initAnns ); var curResults = new List>(); + var states = new HashSet, int>>( + AnonymousEqualityComparer.Create, int>>( + StateKeyEquals, + StateKeyGetHashCode + ) + ); + while (instStack.Count != 0) { DeterministicFsaTraversalInstance inst = instStack.Pop(); @@ -51,6 +61,15 @@ DeterministicFsaTraversalInstance ni in Advance( ) ) { + if (!allMatches) + { + var stateKey = Tuple.Create(ni.State, ni.AnnotationIndex); + if (states.Contains(stateKey)) + { + continue; + } + states.Add(stateKey); + } instStack.Push(ni); } @@ -68,6 +87,23 @@ DeterministicFsaTraversalInstance ni in Advance( return curResults; } + private bool StateKeyEquals( + Tuple, int> x, + Tuple, int> y + ) + { + return x.Item1.Equals(y.Item1) + && x.Item2.Equals(y.Item2); + } + + private int StateKeyGetHashCode(Tuple, int> m) + { + int code = 23; + code = code * 31 + m.Item1.GetHashCode(); + code = code * 31 + m.Item2.GetHashCode(); + return code; + } + protected override DeterministicFsaTraversalInstance CreateInstance() { return new DeterministicFsaTraversalInstance(Fst.RegisterCount); diff --git a/src/SIL.Machine/FiniteState/DeterministicFstTraversalMethod.cs b/src/SIL.Machine/FiniteState/DeterministicFstTraversalMethod.cs index 534a2dcd1..946811c37 100644 --- a/src/SIL.Machine/FiniteState/DeterministicFstTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/DeterministicFstTraversalMethod.cs @@ -26,7 +26,8 @@ public override IEnumerable> Traverse( ref int annIndex, Register[,] initRegisters, IList initCmds, - ISet initAnns + ISet initAnns, + bool allMatches ) { Stack> instStack = InitializeStack( diff --git a/src/SIL.Machine/FiniteState/Fst.cs b/src/SIL.Machine/FiniteState/Fst.cs index 04fb681c4..7eb0af7cc 100644 --- a/src/SIL.Machine/FiniteState/Fst.cs +++ b/src/SIL.Machine/FiniteState/Fst.cs @@ -387,7 +387,7 @@ out IEnumerable> results } List> curResults = traversalMethod - .Traverse(ref annIndex, initRegisters, cmds, initAnns) + .Traverse(ref annIndex, initRegisters, cmds, initAnns, allMatches) .ToList(); if (curResults.Count > 0) { diff --git a/src/SIL.Machine/FiniteState/ITraversalMethod.cs b/src/SIL.Machine/FiniteState/ITraversalMethod.cs index d11dfab46..590e636b3 100644 --- a/src/SIL.Machine/FiniteState/ITraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/ITraversalMethod.cs @@ -11,7 +11,8 @@ IEnumerable> Traverse( ref int annIndex, Register[,] initRegisters, IList initCmds, - ISet initAnns + ISet initAnns, + bool allMatches ); } } diff --git a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs index b5d3b3d5e..c752c32f9 100644 --- a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs @@ -24,7 +24,8 @@ public override IEnumerable> Traverse( ref int annIndex, Register[,] initRegisters, IList initCmds, - ISet initAnns + ISet initAnns, + bool allMatches ) { Stack> instStack = InitializeStack( @@ -35,6 +36,12 @@ ISet initAnns ); var curResults = new List>(); + var states = new HashSet, int>>( + AnonymousEqualityComparer.Create, int>>( + StateKeyEquals, + StateKeyGetHashCode + ) + ); var traversed = new HashSet, int, Register[,]>>( AnonymousEqualityComparer.Create, int, Register[,]>>( KeyEquals, @@ -74,6 +81,15 @@ ISet initAnns arc, curResults ); + if (!allMatches) + { + var stateKey = Tuple.Create(newInst.State, newInst.AnnotationIndex); + if (states.Contains(stateKey)) + { + continue; + } + states.Add(stateKey); + } Tuple, int, Register[,]> key = Tuple.Create( newInst.State, newInst.AnnotationIndex, @@ -109,6 +125,15 @@ NondeterministicFsaTraversalInstance newInst in Advance( ) { newInst.Visited.Clear(); + if (!allMatches) + { + var stateKey = Tuple.Create(newInst.State, newInst.AnnotationIndex); + if (states.Contains(stateKey)) + { + continue; + } + states.Add(stateKey); + } Tuple, int, Register[,]> key = Tuple.Create( newInst.State, newInst.AnnotationIndex, @@ -136,6 +161,22 @@ NondeterministicFsaTraversalInstance newInst in Advance( return curResults; } + private bool StateKeyEquals( + Tuple, int> x, + Tuple, int> y + ) + { + return x.Item1.Equals(y.Item1) + && x.Item2.Equals(y.Item2); + } + + private int StateKeyGetHashCode(Tuple, int> m) + { + int code = 23; + code = code * 31 + m.Item1.GetHashCode(); + code = code * 31 + m.Item2.GetHashCode(); + return code; + } protected override NondeterministicFsaTraversalInstance CreateInstance() { diff --git a/src/SIL.Machine/FiniteState/NondeterministicFstTraversalMethod.cs b/src/SIL.Machine/FiniteState/NondeterministicFstTraversalMethod.cs index e171f4410..aeb45bab9 100644 --- a/src/SIL.Machine/FiniteState/NondeterministicFstTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/NondeterministicFstTraversalMethod.cs @@ -27,7 +27,8 @@ public override IEnumerable> Traverse( ref int annIndex, Register[,] initRegisters, IList initCmds, - ISet initAnns + ISet initAnns, + bool allMatches ) { Stack> instStack = InitializeStack( diff --git a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs index c5934d991..30239539c 100644 --- a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs +++ b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs @@ -91,7 +91,8 @@ public abstract IEnumerable> Traverse( ref int annIndex, Register[,] initRegisters, IList initCmds, - ISet initAnns + ISet initAnns, + bool allMatches ); protected static void ExecuteCommands( From 33c9144944bebf7abfff0ef5db587ea7b465f2ed Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Thu, 17 Sep 2026 09:00:54 -0700 Subject: [PATCH 02/19] Fix format --- .../FiniteState/DeterministicFsaTraversalMethod.cs | 5 +---- .../FiniteState/NondeterministicFsaTraversalMethod.cs | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs index 1be7676ab..0532759f4 100644 --- a/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs @@ -37,10 +37,7 @@ bool allMatches var curResults = new List>(); var states = new HashSet, int>>( - AnonymousEqualityComparer.Create, int>>( - StateKeyEquals, - StateKeyGetHashCode - ) + AnonymousEqualityComparer.Create, int>>(StateKeyEquals, StateKeyGetHashCode) ); while (instStack.Count != 0) diff --git a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs index c752c32f9..776b5be95 100644 --- a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs @@ -37,10 +37,7 @@ bool allMatches var curResults = new List>(); var states = new HashSet, int>>( - AnonymousEqualityComparer.Create, int>>( - StateKeyEquals, - StateKeyGetHashCode - ) + AnonymousEqualityComparer.Create, int>>(StateKeyEquals, StateKeyGetHashCode) ); var traversed = new HashSet, int, Register[,]>>( AnonymousEqualityComparer.Create, int, Register[,]>>( From 3f6ab438a797567cb0d8076b7fd3d8083ba5acaf Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Thu, 17 Sep 2026 10:29:06 -0700 Subject: [PATCH 03/19] Fix format --- .../FiniteState/DeterministicFsaTraversalMethod.cs | 8 ++------ .../FiniteState/NondeterministicFsaTraversalMethod.cs | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs index 0532759f4..2b546810b 100644 --- a/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs @@ -84,13 +84,9 @@ DeterministicFsaTraversalInstance ni in Advance( return curResults; } - private bool StateKeyEquals( - Tuple, int> x, - Tuple, int> y - ) + private bool StateKeyEquals(Tuple, int> x, Tuple, int> y) { - return x.Item1.Equals(y.Item1) - && x.Item2.Equals(y.Item2); + return x.Item1.Equals(y.Item1) && x.Item2.Equals(y.Item2); } private int StateKeyGetHashCode(Tuple, int> m) diff --git a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs index 776b5be95..c78ad76e6 100644 --- a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs @@ -158,13 +158,9 @@ NondeterministicFsaTraversalInstance newInst in Advance( return curResults; } - private bool StateKeyEquals( - Tuple, int> x, - Tuple, int> y - ) + private bool StateKeyEquals(Tuple, int> x, Tuple, int> y) { - return x.Item1.Equals(y.Item1) - && x.Item2.Equals(y.Item2); + return x.Item1.Equals(y.Item1) && x.Item2.Equals(y.Item2); } private int StateKeyGetHashCode(Tuple, int> m) From 1fcd3e71cf7c414b4d14663f5f4c0085802d0e60 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Thu, 17 Sep 2026 10:35:31 -0700 Subject: [PATCH 04/19] Fix format --- .../FiniteState/NondeterministicFsaTraversalMethod.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs index c78ad76e6..9910f49fb 100644 --- a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs @@ -158,6 +158,7 @@ NondeterministicFsaTraversalInstance newInst in Advance( return curResults; } + private bool StateKeyEquals(Tuple, int> x, Tuple, int> y) { return x.Item1.Equals(y.Item1) && x.Item2.Equals(y.Item2); From b348f48fff39491d8f7d3d3b11349260408f996a Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Fri, 18 Sep 2026 07:25:08 -0700 Subject: [PATCH 05/19] Fix bug in how matches are skipped --- .../NondeterministicFsaTraversalMethod.cs | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs index 9910f49fb..addc71b45 100644 --- a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs @@ -78,24 +78,31 @@ bool allMatches arc, curResults ); + bool skip = false; if (!allMatches) { var stateKey = Tuple.Create(newInst.State, newInst.AnnotationIndex); if (states.Contains(stateKey)) { - continue; + skip = true; + } + else + { + states.Add(stateKey); } - states.Add(stateKey); } - Tuple, int, Register[,]> key = Tuple.Create( - newInst.State, - newInst.AnnotationIndex, - newInst.Registers - ); - if (!traversed.Contains(key)) + if (!skip) { - instStack.Push(newInst); - traversed.Add(key); + Tuple, int, Register[,]> key = Tuple.Create( + newInst.State, + newInst.AnnotationIndex, + newInst.Registers + ); + if (!traversed.Contains(key)) + { + instStack.Push(newInst); + traversed.Add(key); + } } if (isInstReusable) releaseInstance = false; From 99900ac2c438f97eb9aa42e41c8cc730ca5106d2 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Fri, 18 Sep 2026 08:22:38 -0700 Subject: [PATCH 06/19] Fix format --- .../FiniteState/NondeterministicFsaTraversalMethod.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs index addc71b45..46a8dedbc 100644 --- a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs @@ -94,9 +94,9 @@ bool allMatches if (!skip) { Tuple, int, Register[,]> key = Tuple.Create( - newInst.State, - newInst.AnnotationIndex, - newInst.Registers + newInst.State, + newInst.AnnotationIndex, + newInst.Registers ); if (!traversed.Contains(key)) { From da352c7a7317da8076b6f7c0211ed491311675ef Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Fri, 18 Sep 2026 08:48:57 -0700 Subject: [PATCH 07/19] Fix format --- .../FiniteState/NondeterministicFsaTraversalMethod.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs index 46a8dedbc..78aed601b 100644 --- a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs @@ -97,7 +97,7 @@ bool allMatches newInst.State, newInst.AnnotationIndex, newInst.Registers - ); + ); if (!traversed.Contains(key)) { instStack.Push(newInst); From e9e8d121d383e7ca8e48c3ac606151430f53a137 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 21 Sep 2026 11:05:49 -0400 Subject: [PATCH 08/19] Add minimal failing repro tests for traversal-dedup match divergence (#516) Two hand-built cases distilled from a 20,000-case differential fuzz (TraversalDedupDifferentialFuzzTests) comparing this branch against master. Both fail here and pass on master: - NondeterministicTraversal_DedupOnVariableBindingLosesMatch: an anchored high=$v0+ match disappears entirely because two instances reach the same (State, AnnotationIndex) with different VariableBindings, and the surviving one can never complete. - DeterministicTraversal_DedupOnRegistersShortensMatch: an alternation match is shortened because two lineages converge on the same (State, AnnotationIndex) with different open-group registers, and the surviving lineage completes earlier than the correct one. Co-authored-by: Claude Sonnet 5 --- .../TraversalDedupMinimalCasesTests.cs | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 tests/SIL.Machine.Tests/Matching/TraversalDedupMinimalCasesTests.cs diff --git a/tests/SIL.Machine.Tests/Matching/TraversalDedupMinimalCasesTests.cs b/tests/SIL.Machine.Tests/Matching/TraversalDedupMinimalCasesTests.cs new file mode 100644 index 000000000..8ce75e4be --- /dev/null +++ b/tests/SIL.Machine.Tests/Matching/TraversalDedupMinimalCasesTests.cs @@ -0,0 +1,134 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.DataStructures; +using SIL.Machine.FeatureModel; + +namespace SIL.Machine.Matching; + +// Minimal, hand-built reproductions of two cases found by a differential fuzz +// (TraversalDedupDifferentialFuzzTests) comparing master against the +// add-allMatches-to-Traverse branch's traversal-dedup change. Both settings +// use AllSubmatches = false and Nondeterministic = false, matching the fuzz. +public class TraversalDedupMinimalCasesTests : PhoneticTestsBase +{ + private FeatureStruct Ann(string voice, string high, string back) + { + return FeatureStruct + .New(PhoneticFeatSys) + .Feature("voice") + .EqualTo(voice) + .Feature("high") + .EqualTo(high) + .Feature("back") + .EqualTo(back) + .Value; + } + + [Test] + public void NondeterministicTraversal_DedupOnVariableBindingLosesMatch() + { + // Pattern: high=$v0+, anchored to both ends (runs NondeterministicFsaTraversalMethod + // because of the variable). The only way to cover the whole input [0,5) with a + // single consistent value of v0 is the run of high- annotations: [0,2)+[2,4)+[4,5). + // Reaching it requires abandoning, at annotation index 0, the parallel instance that + // consumed the high+ annotation [0,1) (which binds v0=+ but then dead-ends, since no + // annotation starts at offset 1). Both instances reach the same (State, AnnotationIndex) + // with different VariableBindings (v0=+ vs v0=-); the traversal-dedup change keys on + // (State, AnnotationIndex) alone and can keep the v0=+ instance, which can never + // complete the anchored match, discarding the one that would have succeeded. + Pattern pattern = Pattern + .New() + .Annotation(FeatureStruct.New(PhoneticFeatSys).Feature("high").EqualToVariable("v0").Value) + .OneOrMore + .Value; + + var data = new AnnotatedStringData(new string('a', 5)); + data.Annotations.Add(0, 2, Ann("voice-", "high-", "back-"), false); + data.Annotations.Add(0, 2, Ann("voice-", "high-", "back-"), false); // duplicate span+values, as the fuzz produced + data.Annotations.Add(0, 1, Ann("voice-", "high+", "back+"), false); + data.Annotations.Add(2, 4, Ann("voice-", "high-", "back+"), false); + data.Annotations.Add(4, 5, Ann("voice+", "high-", "back+"), false); + + var matcher = new Matcher( + pattern, + new MatcherSettings + { + AnchoredToStart = true, + AnchoredToEnd = true, + Direction = Direction.LeftToRight, + AllSubmatches = false, + Nondeterministic = false, + } + ); + + Match match = matcher.Match(data); + + Assert.That(match.Success, Is.True, $"success={match.Success};range={DescribeRange(match.Range)}"); + Assert.That(match.Range, Is.EqualTo(Range.Create(0, 5))); + Assert.That(((FeatureSymbol)match.VariableBindings["v0"]).ID, Is.EqualTo("high-")); + } + + [Test] + public void DeterministicTraversal_DedupOnRegistersShortensMatch() + { + // Pattern: (g0(back=back+) | g1(high=high+ back=back+)), anchored to start only + // (runs DeterministicFsaTraversalMethod: IsDeterministic=True, GroupCount=2). + // The annotation [0,2) satisfies both g0's sole constraint and g1's first constraint, + // so two lineages both consume it as their first step: one has already closed g0 and + // can stop at offset 2, the other still has g1 open, waiting for back=back+ at [2,4). + // They converge on the same (State, AnnotationIndex) with different open-group + // registers. Deduping on (State, AnnotationIndex) alone keeps only one lineage's + // registers - master keeps both and finds the longer g1 match [0,4); the branch's + // surviving lineage yields only the short, earlier-completing g0 match [0,1). + Pattern pattern = Pattern + .New() + .Group("g0", g0 => g0.Annotation(FeatureStruct.New(PhoneticFeatSys).Feature("back").EqualTo("back+").Value)) + .Or.Group( + "g1", + g1 => + g1.Annotation(FeatureStruct.New(PhoneticFeatSys).Feature("high").EqualTo("high+").Value) + .Annotation(FeatureStruct.New(PhoneticFeatSys).Feature("back").EqualTo("back+").Value) + ) + .Value; + + var data = new AnnotatedStringData(new string('a', 4)); + data.Annotations.Add(0, 1, Ann("voice-", "high+", "back+"), false); + data.Annotations.Add(0, 2, Ann("voice-", "high+", "back+"), false); + data.Annotations.Add(2, 4, Ann("voice+", "high-", "back+"), false); + + var matcher = new Matcher( + pattern, + new MatcherSettings + { + AnchoredToStart = true, + AnchoredToEnd = false, + Direction = Direction.LeftToRight, + AllSubmatches = false, + Nondeterministic = false, + } + ); + + Match match = matcher.Match(data); + + Assert.That(match.Success, Is.True); + Assert.That( + match.Range, + Is.EqualTo(Range.Create(0, 4)), + $"success={match.Success};range={DescribeRange(match.Range)};" + + $"g0={DescribeCapture(match.GroupCaptures["g0"])};g1={DescribeCapture(match.GroupCaptures["g1"])}" + ); + Assert.That(match.GroupCaptures["g0"].Success, Is.False); + Assert.That(match.GroupCaptures["g1"].Success, Is.True); + Assert.That(match.GroupCaptures["g1"].Range, Is.EqualTo(Range.Create(0, 4))); + } + + private static string DescribeRange(Range range) + { + return range == Range.Null ? "" : $"[{range.Start},{range.End})"; + } + + private static string DescribeCapture(GroupCapture capture) + { + return capture.Success ? DescribeRange(capture.Range) : ""; + } +} From 091659e7ffe66391bc946691362d448c5af7fc27 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Tue, 22 Sep 2026 09:00:34 -0700 Subject: [PATCH 09/19] Implement two pass traversal with lattice --- .../DeterministicFsaTraversalMethod.cs | 37 ++--- .../NondeterministicFsaTraversalMethod.cs | 59 +------ .../FiniteState/TraversalMethodBase.cs | 156 ++++++++++++++++++ 3 files changed, 178 insertions(+), 74 deletions(-) diff --git a/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs index 2b546810b..9f47d3c3b 100644 --- a/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs @@ -1,8 +1,6 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using SIL.Machine.Annotations; using SIL.Machine.FeatureModel; -using SIL.ObjectModel; namespace SIL.Machine.FiniteState { @@ -36,19 +34,19 @@ bool allMatches ); var curResults = new List>(); - var states = new HashSet, int>>( - AnonymousEqualityComparer.Create, int>>(StateKeyEquals, StateKeyGetHashCode) - ); + var lattice = !allMatches ? CreateFstLattice() : null; while (instStack.Count != 0) { DeterministicFsaTraversalInstance inst = instStack.Pop(); + DeterministicFsaTraversalInstance origInst = !allMatches ? CopyInstance(inst) : null; bool releaseInstance = true; foreach (Arc arc in inst.State.Arcs) { if (CheckInputMatch(arc, inst.AnnotationIndex, inst.VariableBindings)) { + int resultCount = !allMatches ? curResults.Count : 0; foreach ( DeterministicFsaTraversalInstance ni in Advance( inst, @@ -60,12 +58,10 @@ DeterministicFsaTraversalInstance ni in Advance( { if (!allMatches) { - var stateKey = Tuple.Create(ni.State, ni.AnnotationIndex); - if (states.Contains(stateKey)) - { + if (curResults.Count > resultCount) + RecordFinalArc(lattice, origInst, arc); + if (RecordedInstance(lattice, ni, origInst, arc)) continue; - } - states.Add(stateKey); } instStack.Push(ni); } @@ -79,24 +75,17 @@ DeterministicFsaTraversalInstance ni in Advance( ReleaseInstance(inst); } + if (!allMatches) + { + var newResults = ExtractResults(lattice, allMatches); + curResults = newResults; + } + CheckAcceptingStartState(initAnns, initRegisters, curResults); return curResults; } - private bool StateKeyEquals(Tuple, int> x, Tuple, int> y) - { - return x.Item1.Equals(y.Item1) && x.Item2.Equals(y.Item2); - } - - private int StateKeyGetHashCode(Tuple, int> m) - { - int code = 23; - code = code * 31 + m.Item1.GetHashCode(); - code = code * 31 + m.Item2.GetHashCode(); - return code; - } - protected override DeterministicFsaTraversalInstance CreateInstance() { return new DeterministicFsaTraversalInstance(Fst.RegisterCount); diff --git a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs index 78aed601b..378fbd361 100644 --- a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs @@ -36,9 +36,6 @@ bool allMatches ); var curResults = new List>(); - var states = new HashSet, int>>( - AnonymousEqualityComparer.Create, int>>(StateKeyEquals, StateKeyGetHashCode) - ); var traversed = new HashSet, int, Register[,]>>( AnonymousEqualityComparer.Create, int, Register[,]>>( KeyEquals, @@ -78,33 +75,17 @@ bool allMatches arc, curResults ); - bool skip = false; - if (!allMatches) + Tuple, int, Register[,]> key = Tuple.Create( + newInst.State, + newInst.AnnotationIndex, + newInst.Registers + ); + if (!traversed.Contains(key)) { - var stateKey = Tuple.Create(newInst.State, newInst.AnnotationIndex); - if (states.Contains(stateKey)) - { - skip = true; - } - else - { - states.Add(stateKey); - } + instStack.Push(newInst); + traversed.Add(key); } - if (!skip) - { - Tuple, int, Register[,]> key = Tuple.Create( - newInst.State, - newInst.AnnotationIndex, - newInst.Registers - ); - if (!traversed.Contains(key)) - { - instStack.Push(newInst); - traversed.Add(key); - } - } - if (isInstReusable) + if (isInstReusable) releaseInstance = false; varBindings = null; } @@ -129,15 +110,6 @@ NondeterministicFsaTraversalInstance newInst in Advance( ) { newInst.Visited.Clear(); - if (!allMatches) - { - var stateKey = Tuple.Create(newInst.State, newInst.AnnotationIndex); - if (states.Contains(stateKey)) - { - continue; - } - states.Add(stateKey); - } Tuple, int, Register[,]> key = Tuple.Create( newInst.State, newInst.AnnotationIndex, @@ -166,19 +138,6 @@ NondeterministicFsaTraversalInstance newInst in Advance( return curResults; } - private bool StateKeyEquals(Tuple, int> x, Tuple, int> y) - { - return x.Item1.Equals(y.Item1) && x.Item2.Equals(y.Item2); - } - - private int StateKeyGetHashCode(Tuple, int> m) - { - int code = 23; - code = code * 31 + m.Item1.GetHashCode(); - code = code * 31 + m.Item2.GetHashCode(); - return code; - } - protected override NondeterministicFsaTraversalInstance CreateInstance() { return new NondeterministicFsaTraversalInstance(Fst.RegisterCount); diff --git a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs index 30239539c..f976537f9 100644 --- a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs +++ b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using SIL.Extensions; using SIL.Machine.Annotations; using SIL.Machine.DataStructures; using SIL.Machine.FeatureModel; @@ -459,5 +460,160 @@ protected void ReleaseInstance(TInst inst) { _cachedInstances.Enqueue(inst); } + + private readonly Tuple, int> _finalState = new Tuple, int>(null, -1); + + /// + /// Creates a lattice. + /// A lattice is a graph that represents the space of traversals as a packed forest. + /// The nodes are [State, AnnotationIndex] pairs. + /// Each node has a list of incoming arcs that are [Instance, Arc] pairs. + /// The Instance encodes the previous node. + /// + protected IDictionary, int>, IList>>> CreateFstLattice() + { + return new Dictionary, int>, IList>>> + ( + AnonymousEqualityComparer.Create, int>>(StateKeyEquals, StateKeyGetHashCode) + ); + } + + /// + /// Check whether instance is already recorded in lattice. + /// If not, adds instance to lattice. + /// Also adds [origInstance, arc] to instance's incoming arcs. + /// + protected bool RecordedInstance( + IDictionary, int>, IList>>> lattice, + TInst instance, + TInst origInstance, + Arc arc) + { + var stateKey = Tuple.Create(instance.State, instance.AnnotationIndex); + bool recorded = lattice.TryGetValue(stateKey, out IList>> incoming); + if (!recorded) + { + // Add stateKey to lattice. + incoming = new List>>(); + lattice[stateKey] = incoming; + } + // Add [origInstance, arc] to incoming. + incoming.Add(new Tuple>(origInstance, arc)); + return recorded; + } + + protected void RecordFinalArc( + IDictionary, int>, IList>>> lattice, + TInst origInstance, + Arc arc) + { + bool recorded = lattice.TryGetValue(_finalState, out IList>> incoming); + if (!recorded) + { + // Add _finalState to lattice. + incoming = new List>>(); + lattice[_finalState] = incoming; + } + // Add [origInstance, arc] to incoming. + incoming.Add(new Tuple>(origInstance, arc)); + } + + /// + /// Extract the results encoded in lattice under the final state. + /// + protected List> ExtractResults( + IDictionary, int>, IList>>> lattice, + bool allMatches) + { + List> newResults = new List>(); + IList>> incoming; + if (!lattice.TryGetValue(_finalState, out incoming)) + return newResults; + foreach (Tuple> pair in incoming) + { + foreach (TInst instance in ExpandInstances(pair.Item1, lattice, allMatches)) + { + AdvanceInstance(instance, pair.Item2, null, newResults); + } + } + return newResults; + } + + private IList ExpandInstances( + TInst instance, + IDictionary, int>, IList>>> lattice, + bool allMatches) + { + IList instances = new List(); + IList> curResults = new List>(); + var stateKey = Tuple.Create(instance.State, instance.AnnotationIndex); + bool recorded = lattice.TryGetValue(stateKey, out IList>> incoming); + if (!recorded) + { + // The starting instance. + instances.Add(CopyInstance(instance)); + return instances; + } + foreach (Tuple> pair in incoming) + { + foreach (TInst source in ExpandInstances(pair.Item1, lattice, allMatches)) + { + AdvanceInstance(source, pair.Item2, instances, curResults); + } + } + if (false && !allMatches && instances.Count > 1) + { + instances.Sort(InstanceCompare); + TInst first = instances.First(); + instances.Clear(); + instances.Add(first); + } + return instances; + } + + private void AdvanceInstance( + TInst instance, + Arc arc, + IList instances, + IList> curResults) + { + if (CheckInputMatch(arc, instance.AnnotationIndex, instance.VariableBindings)) + { + foreach (TInst ni in Advance(instance, instance.VariableBindings, arc, curResults)) + { + instances?.Add(ni); + } + } + } + + private int InstanceCompare(TInst x, TInst y) + { + int compare = 0; + if (x.Priorities != null) + { + foreach (Tuple priorityPair in x.Priorities.Zip(y.Priorities)) + { + compare = priorityPair.Item1.CompareTo(priorityPair.Item2); + if (compare != 0) + break; + } + } + return compare; + } + + private bool StateKeyEquals(Tuple, int> x, Tuple, int> y) + { + if (x.Item1 == null || y.Item1 == null) + return x.Item1 == y.Item1; + return x.Item1.Equals(y.Item1) && x.Item2.Equals(y.Item2); + } + + private int StateKeyGetHashCode(Tuple, int> m) + { + int code = 23; + code = code * 31 + (m.Item1 != null ? m.Item1.GetHashCode() : 0); + code = code * 31 + m.Item2.GetHashCode(); + return code; + } } } From cd4c7b6e5c70f43e7179706482978ad43d6e1c0e Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Tue, 22 Sep 2026 09:22:00 -0700 Subject: [PATCH 10/19] Fix format --- .../FiniteState/NondeterministicFsaTraversalMethod.cs | 2 +- src/SIL.Machine/FiniteState/TraversalMethodBase.cs | 4 +++- .../Matching/TraversalDedupMinimalCasesTests.cs | 3 +-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs index 378fbd361..11046d0b3 100644 --- a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs @@ -85,7 +85,7 @@ bool allMatches instStack.Push(newInst); traversed.Add(key); } - if (isInstReusable) + if (isInstReusable) releaseInstance = false; varBindings = null; } diff --git a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs index f976537f9..a523c668d 100644 --- a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs +++ b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs @@ -461,7 +461,9 @@ protected void ReleaseInstance(TInst inst) _cachedInstances.Enqueue(inst); } - private readonly Tuple, int> _finalState = new Tuple, int>(null, -1); + private readonly Tuple, int> _finalState = new Tuple, int>( + null, + -1); /// /// Creates a lattice. diff --git a/tests/SIL.Machine.Tests/Matching/TraversalDedupMinimalCasesTests.cs b/tests/SIL.Machine.Tests/Matching/TraversalDedupMinimalCasesTests.cs index 8ce75e4be..4c281463a 100644 --- a/tests/SIL.Machine.Tests/Matching/TraversalDedupMinimalCasesTests.cs +++ b/tests/SIL.Machine.Tests/Matching/TraversalDedupMinimalCasesTests.cs @@ -39,8 +39,7 @@ public void NondeterministicTraversal_DedupOnVariableBindingLosesMatch() Pattern pattern = Pattern .New() .Annotation(FeatureStruct.New(PhoneticFeatSys).Feature("high").EqualToVariable("v0").Value) - .OneOrMore - .Value; + .OneOrMore.Value; var data = new AnnotatedStringData(new string('a', 5)); data.Annotations.Add(0, 2, Ann("voice-", "high-", "back-"), false); From 24d97e9e1284c71cfbb3eb6bf11a12ff90325d86 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Tue, 22 Sep 2026 09:32:45 -0700 Subject: [PATCH 11/19] Fix format --- src/SIL.Machine/FiniteState/TraversalMethodBase.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs index a523c668d..f77e43e10 100644 --- a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs +++ b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs @@ -463,7 +463,8 @@ protected void ReleaseInstance(TInst inst) private readonly Tuple, int> _finalState = new Tuple, int>( null, - -1); + -1 + ); /// /// Creates a lattice. From d5ad117ea2bfbd70e79407346c1c0ab90e758488 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Tue, 22 Sep 2026 09:50:39 -0700 Subject: [PATCH 12/19] Fix format --- .../FiniteState/TraversalMethodBase.cs | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs index f77e43e10..974250ffd 100644 --- a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs +++ b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs @@ -473,15 +473,17 @@ protected void ReleaseInstance(TInst inst) /// Each node has a list of incoming arcs that are [Instance, Arc] pairs. /// The Instance encodes the previous node. /// - protected IDictionary, int>, IList>>> CreateFstLattice() + protected IDictionary< + Tuple, int>, + IList>> + > CreateFstLattice() { - return new Dictionary, int>, IList>>> - ( + return new Dictionary, int>, IList>>>( AnonymousEqualityComparer.Create, int>>(StateKeyEquals, StateKeyGetHashCode) ); } - /// + /// /// Check whether instance is already recorded in lattice. /// If not, adds instance to lattice. /// Also adds [origInstance, arc] to instance's incoming arcs. @@ -490,7 +492,8 @@ protected bool RecordedInstance( IDictionary, int>, IList>>> lattice, TInst instance, TInst origInstance, - Arc arc) + Arc arc + ) { var stateKey = Tuple.Create(instance.State, instance.AnnotationIndex); bool recorded = lattice.TryGetValue(stateKey, out IList>> incoming); @@ -508,7 +511,8 @@ protected bool RecordedInstance( protected void RecordFinalArc( IDictionary, int>, IList>>> lattice, TInst origInstance, - Arc arc) + Arc arc + ) { bool recorded = lattice.TryGetValue(_finalState, out IList>> incoming); if (!recorded) @@ -526,7 +530,8 @@ protected void RecordFinalArc( /// protected List> ExtractResults( IDictionary, int>, IList>>> lattice, - bool allMatches) + bool allMatches + ) { List> newResults = new List>(); IList>> incoming; @@ -545,7 +550,8 @@ protected List> ExtractResults( private IList ExpandInstances( TInst instance, IDictionary, int>, IList>>> lattice, - bool allMatches) + bool allMatches + ) { IList instances = new List(); IList> curResults = new List>(); @@ -578,7 +584,8 @@ private void AdvanceInstance( TInst instance, Arc arc, IList instances, - IList> curResults) + IList> curResults + ) { if (CheckInputMatch(arc, instance.AnnotationIndex, instance.VariableBindings)) { From 95f647ca6da9c8591746708f9da18906bcdecdd5 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Wed, 23 Sep 2026 08:56:38 -0700 Subject: [PATCH 13/19] Fix bugs --- src/SIL.Machine/FiniteState/Fst.cs | 2 +- src/SIL.Machine/FiniteState/TraversalMethodBase.cs | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/SIL.Machine/FiniteState/Fst.cs b/src/SIL.Machine/FiniteState/Fst.cs index 7eb0af7cc..b996b3255 100644 --- a/src/SIL.Machine/FiniteState/Fst.cs +++ b/src/SIL.Machine/FiniteState/Fst.cs @@ -424,7 +424,7 @@ private int ResultCompare(FstResult x, FstResult compare = -compare; if (IsDeterministic) { - compare = x.IsLazy ? -compare : compare; + compare = (x.IsLazy || y.IsLazy) ? -compare : compare; } else if (compare == 0) { diff --git a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs index 974250ffd..730720c33 100644 --- a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs +++ b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs @@ -541,7 +541,7 @@ bool allMatches { foreach (TInst instance in ExpandInstances(pair.Item1, lattice, allMatches)) { - AdvanceInstance(instance, pair.Item2, null, newResults); + AdvanceInstance(instance, pair.Item2, null, newResults, null, 0); } } return newResults; @@ -567,10 +567,10 @@ bool allMatches { foreach (TInst source in ExpandInstances(pair.Item1, lattice, allMatches)) { - AdvanceInstance(source, pair.Item2, instances, curResults); + AdvanceInstance(source, pair.Item2, instances, curResults, instance.State, instance.AnnotationIndex); } } - if (false && !allMatches && instances.Count > 1) + if (!allMatches && instances.Count > 1) { instances.Sort(InstanceCompare); TInst first = instances.First(); @@ -584,14 +584,17 @@ private void AdvanceInstance( TInst instance, Arc arc, IList instances, - IList> curResults + IList> curResults, + State state, + int annotationIndex ) { if (CheckInputMatch(arc, instance.AnnotationIndex, instance.VariableBindings)) { foreach (TInst ni in Advance(instance, instance.VariableBindings, arc, curResults)) { - instances?.Add(ni); + if (instances != null && ni.State == state && ni.AnnotationIndex == annotationIndex) + instances.Add(ni); } } } From 04dd4a92060480a30379d90e138fa99bfaad8423 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Wed, 23 Sep 2026 09:52:06 -0700 Subject: [PATCH 14/19] Fix format --- src/SIL.Machine/FiniteState/TraversalMethodBase.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs index 730720c33..734cead3d 100644 --- a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs +++ b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs @@ -567,7 +567,14 @@ bool allMatches { foreach (TInst source in ExpandInstances(pair.Item1, lattice, allMatches)) { - AdvanceInstance(source, pair.Item2, instances, curResults, instance.State, instance.AnnotationIndex); + AdvanceInstance( + source, + pair.Item2, + instances, + curResults, + instance.State, + instance.AnnotationIndex + ); } } if (!allMatches && instances.Count > 1) From 30a3877f85be624f5cf57316cffc990b32d38aea Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Wed, 23 Sep 2026 14:47:33 -0700 Subject: [PATCH 15/19] Fix unit test --- .../TraversalDedupMinimalCasesTests.cs | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/tests/SIL.Machine.Tests/Matching/TraversalDedupMinimalCasesTests.cs b/tests/SIL.Machine.Tests/Matching/TraversalDedupMinimalCasesTests.cs index 4c281463a..74e96272d 100644 --- a/tests/SIL.Machine.Tests/Matching/TraversalDedupMinimalCasesTests.cs +++ b/tests/SIL.Machine.Tests/Matching/TraversalDedupMinimalCasesTests.cs @@ -70,15 +70,8 @@ public void NondeterministicTraversal_DedupOnVariableBindingLosesMatch() [Test] public void DeterministicTraversal_DedupOnRegistersShortensMatch() { - // Pattern: (g0(back=back+) | g1(high=high+ back=back+)), anchored to start only - // (runs DeterministicFsaTraversalMethod: IsDeterministic=True, GroupCount=2). - // The annotation [0,2) satisfies both g0's sole constraint and g1's first constraint, - // so two lineages both consume it as their first step: one has already closed g0 and - // can stop at offset 2, the other still has g1 open, waiting for back=back+ at [2,4). - // They converge on the same (State, AnnotationIndex) with different open-group - // registers. Deduping on (State, AnnotationIndex) alone keeps only one lineage's - // registers - master keeps both and finds the longer g1 match [0,4); the branch's - // surviving lineage yields only the short, earlier-completing g0 match [0,1). + // This verifies that the bug where ResultCompare is asymmetric has been fixed. + // When a pattern gets determinized and there is an alternative, the shortest path is preferred. Pattern pattern = Pattern .New() .Group("g0", g0 => g0.Annotation(FeatureStruct.New(PhoneticFeatSys).Feature("back").EqualTo("back+").Value)) @@ -112,13 +105,13 @@ public void DeterministicTraversal_DedupOnRegistersShortensMatch() Assert.That(match.Success, Is.True); Assert.That( match.Range, - Is.EqualTo(Range.Create(0, 4)), + Is.EqualTo(Range.Create(0, 1)), $"success={match.Success};range={DescribeRange(match.Range)};" + $"g0={DescribeCapture(match.GroupCaptures["g0"])};g1={DescribeCapture(match.GroupCaptures["g1"])}" ); - Assert.That(match.GroupCaptures["g0"].Success, Is.False); - Assert.That(match.GroupCaptures["g1"].Success, Is.True); - Assert.That(match.GroupCaptures["g1"].Range, Is.EqualTo(Range.Create(0, 4))); + Assert.That(match.GroupCaptures["g0"].Success, Is.True); + Assert.That(match.GroupCaptures["g1"].Success, Is.False); + Assert.That(match.GroupCaptures["g1"].Range, Is.EqualTo(Range.Create(-1, -1))); } private static string DescribeRange(Range range) From 225f3087e14a12a05333a8af8b041896d82a3957 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Thu, 24 Sep 2026 10:52:57 -0700 Subject: [PATCH 16/19] Add VariableBindings to LatticeNodeKey; add lattice to Nondeterministic --- .../DeterministicFsaTraversalMethod.cs | 2 +- .../NondeterministicFsaTraversalMethod.cs | 50 +++++++--- .../FiniteState/TraversalMethodBase.cs | 92 +++++++++++++------ 3 files changed, 103 insertions(+), 41 deletions(-) diff --git a/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs index 9f47d3c3b..e3d9a04d5 100644 --- a/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs @@ -39,7 +39,7 @@ bool allMatches while (instStack.Count != 0) { DeterministicFsaTraversalInstance inst = instStack.Pop(); - DeterministicFsaTraversalInstance origInst = !allMatches ? CopyInstance(inst) : null; + DeterministicFsaTraversalInstance origInst = !allMatches ? CopyInstanceAndBindings(inst) : null; bool releaseInstance = true; foreach (Arc arc in inst.State.Arcs) diff --git a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs index 11046d0b3..59cd613f1 100644 --- a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs @@ -36,6 +36,7 @@ bool allMatches ); var curResults = new List>(); + var lattice = !allMatches ? CreateFstLattice() : null; var traversed = new HashSet, int, Register[,]>>( AnonymousEqualityComparer.Create, int, Register[,]>>( KeyEquals, @@ -45,6 +46,7 @@ bool allMatches while (instStack.Count != 0) { NondeterministicFsaTraversalInstance inst = instStack.Pop(); + NondeterministicFsaTraversalInstance origInst = !allMatches ? CopyInstanceAndBindings(inst) : null; bool releaseInstance = true; VariableBindings varBindings = null; @@ -70,24 +72,36 @@ bool allMatches } ti.Visited.Add(arc.Target); + int resultCount = !allMatches ? curResults.Count : 0; NondeterministicFsaTraversalInstance newInst = EpsilonAdvance( ti, arc, curResults ); - Tuple, int, Register[,]> key = Tuple.Create( - newInst.State, - newInst.AnnotationIndex, - newInst.Registers - ); - if (!traversed.Contains(key)) + bool skip = false; + if (!allMatches) { - instStack.Push(newInst); - traversed.Add(key); + if (curResults.Count > resultCount) + RecordFinalArc(lattice, origInst, arc); + if (RecordedInstance(lattice, newInst, origInst, arc)) + skip = true; + } + if (!skip) + { + Tuple, int, Register[,]> key = Tuple.Create( + newInst.State, + newInst.AnnotationIndex, + newInst.Registers + ); + if (!traversed.Contains(key)) + { + instStack.Push(newInst); + traversed.Add(key); + } + if (isInstReusable) + releaseInstance = false; + varBindings = null; } - if (isInstReusable) - releaseInstance = false; - varBindings = null; } } else @@ -100,6 +114,7 @@ bool allMatches ? inst : CopyInstance(inst); + int resultCount = !allMatches ? curResults.Count : 0; foreach ( NondeterministicFsaTraversalInstance newInst in Advance( ti, @@ -110,6 +125,13 @@ NondeterministicFsaTraversalInstance newInst in Advance( ) { newInst.Visited.Clear(); + if (!allMatches) + { + if (curResults.Count > resultCount) + RecordFinalArc(lattice, origInst, arc); + if (RecordedInstance(lattice, newInst, origInst, arc)) + continue; + } Tuple, int, Register[,]> key = Tuple.Create( newInst.State, newInst.AnnotationIndex, @@ -133,6 +155,12 @@ NondeterministicFsaTraversalInstance newInst in Advance( ReleaseInstance(inst); } + if (!allMatches) + { + var newResults = ExtractResults(lattice, allMatches); + curResults = newResults; + } + CheckAcceptingStartState(initAnns, initRegisters, curResults); return curResults; diff --git a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs index 734cead3d..cc511ee6e 100644 --- a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs +++ b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs @@ -454,6 +454,16 @@ protected TInst CopyInstance(TInst inst) return ni; } + protected TInst CopyInstanceAndBindings(TInst inst) + { + TInst ni = CopyInstance(inst); + if (inst.VariableBindings != null) + { + ni.VariableBindings = inst.VariableBindings.Clone(); + } + return ni; + } + protected abstract TInst CreateInstance(); protected void ReleaseInstance(TInst inst) @@ -461,10 +471,14 @@ protected void ReleaseInstance(TInst inst) _cachedInstances.Enqueue(inst); } - private readonly Tuple, int> _finalState = new Tuple, int>( - null, - -1 - ); + protected class LatticeNodeKey + { + public State State { get; set; } + public int AnnotationIndex { get; set; } + public VariableBindings VariableBindings { get; set; } + } + + private readonly LatticeNodeKey _finalState = new LatticeNodeKey(); /// /// Creates a lattice. @@ -473,13 +487,10 @@ protected void ReleaseInstance(TInst inst) /// Each node has a list of incoming arcs that are [Instance, Arc] pairs. /// The Instance encodes the previous node. /// - protected IDictionary< - Tuple, int>, - IList>> - > CreateFstLattice() + protected IDictionary>>> CreateFstLattice() { - return new Dictionary, int>, IList>>>( - AnonymousEqualityComparer.Create, int>>(StateKeyEquals, StateKeyGetHashCode) + return new Dictionary>>>( + AnonymousEqualityComparer.Create(LatticeNodeKeyEquals, LatticeNodeKeyGetHashCode) ); } @@ -489,19 +500,24 @@ protected IDictionary< /// Also adds [origInstance, arc] to instance's incoming arcs. /// protected bool RecordedInstance( - IDictionary, int>, IList>>> lattice, + IDictionary>>> lattice, TInst instance, TInst origInstance, Arc arc ) { - var stateKey = Tuple.Create(instance.State, instance.AnnotationIndex); - bool recorded = lattice.TryGetValue(stateKey, out IList>> incoming); + var nodeKey = new LatticeNodeKey() + { + State = instance.State, + AnnotationIndex = instance.AnnotationIndex, + VariableBindings = instance.VariableBindings?.Clone(), + }; + bool recorded = lattice.TryGetValue(nodeKey, out IList>> incoming); if (!recorded) { - // Add stateKey to lattice. + // Add nodeKey to lattice. incoming = new List>>(); - lattice[stateKey] = incoming; + lattice[nodeKey] = incoming; } // Add [origInstance, arc] to incoming. incoming.Add(new Tuple>(origInstance, arc)); @@ -509,7 +525,7 @@ Arc arc } protected void RecordFinalArc( - IDictionary, int>, IList>>> lattice, + IDictionary>>> lattice, TInst origInstance, Arc arc ) @@ -529,7 +545,7 @@ Arc arc /// Extract the results encoded in lattice under the final state. /// protected List> ExtractResults( - IDictionary, int>, IList>>> lattice, + IDictionary>>> lattice, bool allMatches ) { @@ -549,18 +565,23 @@ bool allMatches private IList ExpandInstances( TInst instance, - IDictionary, int>, IList>>> lattice, + IDictionary>>> lattice, bool allMatches ) { IList instances = new List(); IList> curResults = new List>(); - var stateKey = Tuple.Create(instance.State, instance.AnnotationIndex); - bool recorded = lattice.TryGetValue(stateKey, out IList>> incoming); + LatticeNodeKey nodeKey = new LatticeNodeKey() + { + State = instance.State, + AnnotationIndex = instance.AnnotationIndex, + VariableBindings = instance.VariableBindings?.Clone(), + }; + bool recorded = lattice.TryGetValue(nodeKey, out IList>> incoming); if (!recorded) { // The starting instance. - instances.Add(CopyInstance(instance)); + instances.Add(CopyInstanceAndBindings(instance)); return instances; } foreach (Tuple> pair in incoming) @@ -596,7 +617,15 @@ private void AdvanceInstance( int annotationIndex ) { - if (CheckInputMatch(arc, instance.AnnotationIndex, instance.VariableBindings)) + if (arc.Input.IsEpsilon) + { + TInst ni = EpsilonAdvance(instance, arc, curResults); + if (instances != null && ni.State == state && ni.AnnotationIndex == annotationIndex) + { + instances.Add(ni); + } + } + else if (CheckInputMatch(arc, instance.AnnotationIndex, instance.VariableBindings)) { foreach (TInst ni in Advance(instance, instance.VariableBindings, arc, curResults)) { @@ -621,18 +650,23 @@ private int InstanceCompare(TInst x, TInst y) return compare; } - private bool StateKeyEquals(Tuple, int> x, Tuple, int> y) + private bool LatticeNodeKeyEquals(LatticeNodeKey x, LatticeNodeKey y) { - if (x.Item1 == null || y.Item1 == null) - return x.Item1 == y.Item1; - return x.Item1.Equals(y.Item1) && x.Item2.Equals(y.Item2); + if (x.State == null || y.State == null) + return x.State == y.State; + if (x.VariableBindings == null) + return x.State.Equals(y.State) && x.AnnotationIndex.Equals(y.AnnotationIndex); + return x.State.Equals(y.State) + && x.AnnotationIndex.Equals(y.AnnotationIndex) + && x.VariableBindings.Equals(y.VariableBindings); } - private int StateKeyGetHashCode(Tuple, int> m) + private int LatticeNodeKeyGetHashCode(LatticeNodeKey m) { int code = 23; - code = code * 31 + (m.Item1 != null ? m.Item1.GetHashCode() : 0); - code = code * 31 + m.Item2.GetHashCode(); + code = code * 31 + (m.State != null ? m.State.GetHashCode() : 0); + code = code * 31 + m.AnnotationIndex.GetHashCode(); + code = code * 31 + (m.VariableBindings != null ? m.VariableBindings.GetHashCode() : 0); return code; } } From 14a13a890e440f3522c7a32f4e1d4c1f48d6ab13 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Thu, 24 Sep 2026 11:57:22 -0700 Subject: [PATCH 17/19] Cache lattice arc results --- .../FiniteState/TraversalMethodBase.cs | 79 ++++++++++++------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs index cc511ee6e..b327659dd 100644 --- a/src/SIL.Machine/FiniteState/TraversalMethodBase.cs +++ b/src/SIL.Machine/FiniteState/TraversalMethodBase.cs @@ -471,14 +471,21 @@ protected void ReleaseInstance(TInst inst) _cachedInstances.Enqueue(inst); } - protected class LatticeNodeKey + protected class LatticeNode { public State State { get; set; } public int AnnotationIndex { get; set; } public VariableBindings VariableBindings { get; set; } } - private readonly LatticeNodeKey _finalState = new LatticeNodeKey(); + protected class LatticeArc + { + public TInst Instance { get; set; } + public Arc Arc { get; set; } + public IList Instances { get; set; } + } + + private readonly LatticeNode _finalState = new LatticeNode(); /// /// Creates a lattice. @@ -487,10 +494,10 @@ protected class LatticeNodeKey /// Each node has a list of incoming arcs that are [Instance, Arc] pairs. /// The Instance encodes the previous node. /// - protected IDictionary>>> CreateFstLattice() + protected IDictionary> CreateFstLattice() { - return new Dictionary>>>( - AnonymousEqualityComparer.Create(LatticeNodeKeyEquals, LatticeNodeKeyGetHashCode) + return new Dictionary>( + AnonymousEqualityComparer.Create(LatticeNodeKeyEquals, LatticeNodeKeyGetHashCode) ); } @@ -500,97 +507,115 @@ protected IDictionary>>> /// Also adds [origInstance, arc] to instance's incoming arcs. /// protected bool RecordedInstance( - IDictionary>>> lattice, + IDictionary> lattice, TInst instance, TInst origInstance, Arc arc ) { - var nodeKey = new LatticeNodeKey() + var nodeKey = new LatticeNode() { State = instance.State, AnnotationIndex = instance.AnnotationIndex, VariableBindings = instance.VariableBindings?.Clone(), }; - bool recorded = lattice.TryGetValue(nodeKey, out IList>> incoming); + bool recorded = lattice.TryGetValue(nodeKey, out IList incoming); if (!recorded) { // Add nodeKey to lattice. - incoming = new List>>(); + incoming = new List(); lattice[nodeKey] = incoming; } // Add [origInstance, arc] to incoming. - incoming.Add(new Tuple>(origInstance, arc)); + incoming.Add(new LatticeArc() { Instance = origInstance, Arc = arc }); return recorded; } protected void RecordFinalArc( - IDictionary>>> lattice, + IDictionary> lattice, TInst origInstance, Arc arc ) { - bool recorded = lattice.TryGetValue(_finalState, out IList>> incoming); + bool recorded = lattice.TryGetValue(_finalState, out IList incoming); if (!recorded) { // Add _finalState to lattice. - incoming = new List>>(); + incoming = new List(); lattice[_finalState] = incoming; } // Add [origInstance, arc] to incoming. - incoming.Add(new Tuple>(origInstance, arc)); + incoming.Add(new LatticeArc() { Instance = origInstance, Arc = arc }); } /// /// Extract the results encoded in lattice under the final state. /// protected List> ExtractResults( - IDictionary>>> lattice, + IDictionary> lattice, bool allMatches ) { List> newResults = new List>(); - IList>> incoming; + IList incoming; if (!lattice.TryGetValue(_finalState, out incoming)) return newResults; - foreach (Tuple> pair in incoming) + foreach (LatticeArc latticeArc in incoming) { - foreach (TInst instance in ExpandInstances(pair.Item1, lattice, allMatches)) + foreach (TInst instance in ExpandArcInstances(latticeArc, lattice, allMatches)) { - AdvanceInstance(instance, pair.Item2, null, newResults, null, 0); + AdvanceInstance(instance, latticeArc.Arc, null, newResults, null, 0); } } return newResults; } + private IList ExpandArcInstances( + LatticeArc latticeArc, + IDictionary> lattice, + bool allMatches + ) + { + if (latticeArc.Instances == null) + { + latticeArc.Instances = ExpandInstances(latticeArc.Instance, lattice, allMatches); + } + IList instances = new List(); + foreach (TInst instance in latticeArc.Instances) + { + instances.Add(CopyInstanceAndBindings(instance)); + } + return instances; + } + private IList ExpandInstances( TInst instance, - IDictionary>>> lattice, + IDictionary> lattice, bool allMatches ) { IList instances = new List(); IList> curResults = new List>(); - LatticeNodeKey nodeKey = new LatticeNodeKey() + LatticeNode nodeKey = new LatticeNode() { State = instance.State, AnnotationIndex = instance.AnnotationIndex, VariableBindings = instance.VariableBindings?.Clone(), }; - bool recorded = lattice.TryGetValue(nodeKey, out IList>> incoming); + bool recorded = lattice.TryGetValue(nodeKey, out IList incoming); if (!recorded) { // The starting instance. instances.Add(CopyInstanceAndBindings(instance)); return instances; } - foreach (Tuple> pair in incoming) + foreach (LatticeArc latticeArc in incoming) { - foreach (TInst source in ExpandInstances(pair.Item1, lattice, allMatches)) + foreach (TInst source in ExpandArcInstances(latticeArc, lattice, allMatches)) { AdvanceInstance( source, - pair.Item2, + latticeArc.Arc, instances, curResults, instance.State, @@ -650,7 +675,7 @@ private int InstanceCompare(TInst x, TInst y) return compare; } - private bool LatticeNodeKeyEquals(LatticeNodeKey x, LatticeNodeKey y) + private bool LatticeNodeKeyEquals(LatticeNode x, LatticeNode y) { if (x.State == null || y.State == null) return x.State == y.State; @@ -661,7 +686,7 @@ private bool LatticeNodeKeyEquals(LatticeNodeKey x, LatticeNodeKey y) && x.VariableBindings.Equals(y.VariableBindings); } - private int LatticeNodeKeyGetHashCode(LatticeNodeKey m) + private int LatticeNodeKeyGetHashCode(LatticeNode m) { int code = 23; code = code * 31 + (m.State != null ? m.State.GetHashCode() : 0); From 2907bfb55373182945e069d05598be57bf54c0e2 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Thu, 24 Sep 2026 13:02:58 -0700 Subject: [PATCH 18/19] Fix format --- .../FiniteState/DeterministicFsaTraversalMethod.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs index e3d9a04d5..e06a6ad0a 100644 --- a/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/DeterministicFsaTraversalMethod.cs @@ -39,7 +39,9 @@ bool allMatches while (instStack.Count != 0) { DeterministicFsaTraversalInstance inst = instStack.Pop(); - DeterministicFsaTraversalInstance origInst = !allMatches ? CopyInstanceAndBindings(inst) : null; + DeterministicFsaTraversalInstance origInst = !allMatches + ? CopyInstanceAndBindings(inst) + : null; bool releaseInstance = true; foreach (Arc arc in inst.State.Arcs) From 8a51794a407def811500087529ad54af048d89d2 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Thu, 24 Sep 2026 13:15:41 -0700 Subject: [PATCH 19/19] Fix format --- .../FiniteState/NondeterministicFsaTraversalMethod.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs index 59cd613f1..f1488c802 100644 --- a/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs +++ b/src/SIL.Machine/FiniteState/NondeterministicFsaTraversalMethod.cs @@ -46,7 +46,9 @@ bool allMatches while (instStack.Count != 0) { NondeterministicFsaTraversalInstance inst = instStack.Pop(); - NondeterministicFsaTraversalInstance origInst = !allMatches ? CopyInstanceAndBindings(inst) : null; + NondeterministicFsaTraversalInstance origInst = !allMatches + ? CopyInstanceAndBindings(inst) + : null; bool releaseInstance = true; VariableBindings varBindings = null;