diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index f55be0b49..b972790ad 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -61,6 +61,7 @@ public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParalle MaxStemCount = 2; MaxAlternatives = 0; MergeEquivalentAnalyses = true; + PruneDisagreeingCopies = true; LexEntrySelector = entry => true; RuleSelector = rule => true; @@ -88,6 +89,13 @@ public ITraceManager TraceManager /// public bool MergeEquivalentAnalyses { get; set; } + /// + /// When unapplying a rule that copies a part more than once (reduplication), skip matches whose + /// copies cannot unify segment by segment. Such a match can never survive synthesis, so this removes + /// only doomed analyses. On by default. + /// + public bool PruneDisagreeingCopies { get; set; } + /// /// Caps the concurrency used within a single parse or generation -- analysis cascade, /// affix-template unapplication and synthesis alike. A value of 1 runs the work fully diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs index 52e57be3d..3d0ed466a 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs @@ -29,7 +29,8 @@ public AnalysisAffixProcessRule(Morpher morpher, AffixProcessRule rule) foreach (AffixProcessAllomorph allo in rule.Allomorphs) { _rules.Add( - new MultiplePatternRule( + new CopyAgreementPatternRule( + morpher, new AnalysisAffixProcessAllomorphRuleSpec(allo), new MatcherSettings { diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs index 93e7a0ef6..90948926d 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs @@ -46,6 +46,82 @@ protected IDictionary CapturedParts get { return _capturedParts; } } + internal bool HasRepeatedParts + { + get { return _capturedParts.Values.Any(count => count >= 2); } + } + + /// + /// Synthesis writes every copy of a part from the same input, and anything that later changes one + /// copy is unapplied before this rule, so copies proven to disagree segment by segment cannot lead + /// to a valid analysis. A copy containing an optional node (an unapplied deletion), or a part the + /// rule modifies, cannot be judged and never counts as disagreeing. + /// + internal bool HasDisagreeingCopies(Match match) + { + foreach (KeyValuePair capturedPart in _capturedParts) + { + string partName = capturedPart.Key; + int copyCount = capturedPart.Value; + if (copyCount < 2) + continue; + + if (_modifyFromInfos.ContainsKey(partName)) + continue; + + var copies = new List>(copyCount); + bool partUndecidable = false; + for (int i = 0; i < copyCount; i++) + { + GroupCapture capture = match.GroupCaptures[GetGroupName(partName, i)]; + if (!capture.Success) + { + partUndecidable = true; + break; + } + + List nodes = GetCapturedNodes(match.Input.Shape, capture.Range); + if (nodes.Any(node => node.Annotation.Optional)) + { + partUndecidable = true; + break; + } + + copies.Add(nodes.Where(node => node.Annotation.Type() == HCFeatureSystem.Segment).ToList()); + } + + if (partUndecidable) + continue; + + List firstCopy = copies[0]; + for (int i = 1; i < copies.Count; i++) + { + List otherCopy = copies[i]; + if (firstCopy.Count != otherCopy.Count) + return true; + + for (int j = 0; j < firstCopy.Count; j++) + { + if (!firstCopy[j].Annotation.FeatureStruct.IsUnifiable(otherCopy[j].Annotation.FeatureStruct)) + return true; + } + } + } + + return false; + } + + /// + /// Includes optional nodes the matcher skipped before the capture when they reach the start of the + /// shape, so an unapplied word-initial deletion is still seen. + /// + private static List GetCapturedNodes(Shape shape, Range range) + { + var nodes = new List(MorphologicalOutputAction.SkippedOptionalNodes(shape, range)); + nodes.AddRange(shape.GetNodes(range)); + return nodes; + } + public Pattern Pattern { get { return _pattern; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs index 749aa7b4d..cf4281c70 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs @@ -22,7 +22,8 @@ public AnalysisRealizationalAffixProcessRule(Morpher morpher, RealizationalAffix foreach (AffixProcessAllomorph allo in rule.Allomorphs) { _rules.Add( - new MultiplePatternRule( + new CopyAgreementPatternRule( + morpher, new AnalysisAffixProcessAllomorphRuleSpec(allo), new MatcherSettings { diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs new file mode 100644 index 000000000..78abce1ff --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs @@ -0,0 +1,40 @@ +using System.Collections.Generic; +using SIL.Machine.Annotations; +using SIL.Machine.Matching; +using SIL.Machine.Rules; + +namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules +{ + internal sealed class CopyAgreementPatternRule : MultiplePatternRule + { + private readonly Morpher _morpher; + private readonly AnalysisAffixProcessAllomorphRuleSpec _spec; + private readonly bool _hasRepeatedParts; + + public CopyAgreementPatternRule( + Morpher morpher, + AnalysisAffixProcessAllomorphRuleSpec ruleSpec, + MatcherSettings matcherSettings + ) + : base(ruleSpec, matcherSettings) + { + _morpher = morpher; + _spec = ruleSpec; + _hasRepeatedParts = ruleSpec.HasRepeatedParts; + } + + protected override IEnumerable ApplyImpl(Word input, ShapeNode start) + { + bool prune = _morpher.PruneDisagreeingCopies && _hasRepeatedParts; + var results = new List(); + foreach (Match match in Matcher.AllMatches(input, start)) + { + if (prune && _spec.HasDisagreeingCopies(match)) + continue; + + results.Add(RuleSpec.ApplyRhs(this, match)); + } + return results; + } + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs index 5595a232f..508a458c0 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs @@ -38,6 +38,11 @@ IDictionary capturedParts public abstract IEnumerable> Apply(Match match, Word output); protected IEnumerable GetSkippedOptionalNodes(Shape shape, Range range) + { + return SkippedOptionalNodes(shape, range); + } + + internal static IEnumerable SkippedOptionalNodes(Shape shape, Range range) { ShapeNode node = range.Start.Prev; var skippedNodes = new List(); diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AffixProcessRuleTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AffixProcessRuleTests.cs index 329074617..579d997db 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AffixProcessRuleTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AffixProcessRuleTests.cs @@ -965,8 +965,9 @@ public void SimulfixRules() AssertMorphsEqual(morpher.ParseWord("sɯɯpu"), "50 SIMUL"); } - [Test] - public void ReduplicationRules() + [TestCase(false)] + [TestCase(true)] + public void ReduplicationRules(bool pruneCopies) { var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; var cons = FeatureStruct @@ -1015,7 +1016,7 @@ public void ReduplicationRules() ); Morphophonemic.MorphologicalRules.Add(redup); - var morpher = new Morpher(TraceManager, Language); + var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("sasag"), "RED 32"); var voicing = new RewriteRule @@ -1033,7 +1034,7 @@ public void ReduplicationRules() ); Allophonic.PhonologicalRules.Add(voicing); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("sazag"), "RED 32"); var affrication = new RewriteRule @@ -1050,7 +1051,7 @@ public void ReduplicationRules() ); Allophonic.PhonologicalRules.Add(affrication); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("tsazag"), "RED 32"); redup.Allomorphs.Clear(); @@ -1066,7 +1067,7 @@ public void ReduplicationRules() } ); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("tsagag"), "32 RED"); Allophonic.PhonologicalRules.Clear(); @@ -1084,7 +1085,7 @@ public void ReduplicationRules() } ); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("sagsag"), "32 RED"); AssertMorphsEqual(morpher.ParseWord("sasibudbud"), "38 RED"); @@ -1115,7 +1116,7 @@ public void ReduplicationRules() ); Allophonic.PhonologicalRules.Add(gDelete); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("saag"), "32 RED"); gDelete.Subrules.Clear(); @@ -1126,7 +1127,7 @@ public void ReduplicationRules() } ); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("saga"), "32 RED"); Allophonic.PhonologicalRules.Clear(); @@ -1152,7 +1153,7 @@ public void ReduplicationRules() } ); - morpher = new Morpher(TraceManager, Language); + morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("buiuib"), "30 RED", "31 RED"); } @@ -1900,8 +1901,9 @@ public void SubsumedAffix() AssertMorphsEqual(morpher.ParseWord("bubib"), "42 PRES", "43 PRES"); } - [Test] - public void ModifyFromInputRules() + [TestCase(false)] + [TestCase(true)] + public void ModifyFromInputRules(bool pruneCopies) { var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; var vowel = FeatureStruct @@ -1941,7 +1943,7 @@ public void ModifyFromInputRules() ); Morphophonemic.MorphologicalRules.Add(sSuffix); - var morpher = new Morpher(TraceManager, Language); + var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; AssertMorphsEqual(morpher.ParseWord("puso"), "52 PL"); } diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs new file mode 100644 index 000000000..5f9a5d74c --- /dev/null +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/CopyAgreementPruneTests.cs @@ -0,0 +1,153 @@ +using NUnit.Framework; +using SIL.Machine.Annotations; +using SIL.Machine.FeatureModel; +using SIL.Machine.Matching; +using SIL.Machine.Morphology.HermitCrab.PhonologicalRules; + +namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules; + +[TestFixture] +public class CopyAgreementPruneTests : HermitCrabTestBase +{ + [Test] + public void PruningIsOnByDefault() + { + Assert.That(new Morpher(TraceManager, Language).PruneDisagreeingCopies, Is.True); + } + + [TestCase(false, 3)] + [TestCase(true, 1)] + public void FullCopyKeepsOnlyTheSplitWhoseCopiesAgree(bool pruneCopies, int expectedOutputs) + { + (string first, string second) = FindIncompatibleSegments(); + var anySegment = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + AffixProcessRule rule = CreateCopyRule( + Pattern.New("copy").Annotation(anySegment).OneOrMore.Value + ); + var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; + + List outputs = new AnalysisAffixProcessRule(morpher, rule) + .Apply(CreateInput(first + second + first + second)) + .ToList(); + + Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); + if (pruneCopies) + Assert.That(outputs.Single().Shape.Count, Is.EqualTo(2)); + } + + [TestCase(false, 1)] + [TestCase(true, 0)] + public void DisagreeingCopiesAreRemovedOnlyWhenPruning(bool pruneCopies, int expectedOutputs) + { + (string first, string second) = FindIncompatibleSegments(); + var anySegment = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + AffixProcessRule rule = CreateCopyRule(Pattern.New("copy").Annotation(anySegment).Value); + var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies }; + + List outputs = new AnalysisAffixProcessRule(morpher, rule).Apply(CreateInput(first + second)).ToList(); + + Assert.That(outputs.Count, Is.EqualTo(expectedOutputs)); + } + + /// + /// The prune's weakest point: a self-feeding deletion strips two segments from the second copy only, + /// and = 0 lets analysis restore just one of them. + /// + [TestCase(0)] + [TestCase(1)] + public void CapLimitedDeletionInOneCopyParsesTheSameWithAndWithoutPruning(int deletionReapplications) + { + var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + var vowel = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("voc+") + .Value; + AffixProcessRule redup = CreateCopyRule(Pattern.New("copy").Annotation(any).OneOrMore.Value); + redup.RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value; + Morphophonemic.MorphologicalRules.Add(redup); + + var gDelete = new RewriteRule + { + Name = "g_delete", + Lhs = Pattern.New().Annotation(Character(Table1, "g")).Value, + }; + gDelete.Subrules.Add( + new RewriteSubrule { LeftEnvironment = Pattern.New().Annotation(vowel).Value } + ); + Allophonic.PhonologicalRules.Add(gDelete); + + LexEntry root = AddEntry( + "GG", + FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + Morphophonemic, + "ggasa" + ); + try + { + var generator = new Morpher(TraceManager, Language); + Assert.That( + generator.GenerateWords(root, new Morpheme[] { redup }, FeatureStruct.New().Value), + Is.EquivalentTo(new[] { "ggasaasa" }), + "forward derivation defines the expected parse" + ); + + string[] Parse(bool pruneCopies) + { + var morpher = new Morpher(TraceManager, Language) + { + DeletionReapplications = deletionReapplications, + PruneDisagreeingCopies = pruneCopies, + }; + return morpher + .ParseWord("ggasaasa") + .Select(w => string.Join(" ", w.AllomorphsInMorphOrder.Select(a => a.Morpheme.Gloss))) + .OrderBy(s => s, StringComparer.Ordinal) + .ToArray(); + } + + string[] unpruned = Parse(false); + Assert.That(unpruned, Does.Contain("GG RED"), "the unpruned engine finds the generated analysis"); + Assert.That(Parse(true), Is.EqualTo(unpruned)); + } + finally + { + Morphophonemic.Entries.Remove(root); + } + } + + private static AffixProcessRule CreateCopyRule(Pattern part) + { + var rule = new AffixProcessRule + { + Name = "full_copy", + Gloss = "RED", + RequiredSyntacticFeatureStruct = FeatureStruct.New().Value, + OutSyntacticFeatureStruct = FeatureStruct.New().Value, + }; + rule.Allomorphs.Add( + new AffixProcessAllomorph { Lhs = { part }, Rhs = { new CopyFromInput("copy"), new CopyFromInput("copy") } } + ); + return rule; + } + + private (string First, string Second) FindIncompatibleSegments() + { + CharacterDefinition[] segments = Table3 + .Where(definition => definition.Type == HCFeatureSystem.Segment) + .ToArray(); + var pair = segments + .SelectMany( + (first, index) => segments.Skip(index + 1).Select(second => new { First = first, Second = second }) + ) + .First(candidate => !candidate.First.FeatureStruct.IsUnifiable(candidate.Second.FeatureStruct)); + return (pair.First.Representations.First(), pair.Second.Representations.First()); + } + + private Word CreateInput(string representation) + { + var input = new Word(Surface, Table3.Segment(representation)) { AnalysisScope = new AnalysisScope() }; + input.Freeze(); + return input; + } +}