Prune reduplication matches whose copies cannot agree

src/SIL.Machine.Morphology.HermitCrab/Morpher.cs

@@ -61,6 +61,7 @@ public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParalle
             MaxStemCount = 2;
             MaxAlternatives = 0;
             MergeEquivalentAnalyses = true;
+            PruneDisagreeingCopies = true;
             LexEntrySelector = entry => true;
             RuleSelector = rule => true;
 
@@ -88,6 +89,13 @@ public ITraceManager TraceManager
         /// </summary>
         public bool MergeEquivalentAnalyses { get; set; }
 
+        /// <summary>
+        /// When unapplying a rule that copies a part more than once (reduplication), skip matches whose
+        /// copies cannot unify segment by segment. Such a match can never survive synthesis, so this removes
+        /// only doomed analyses. On by default.
+        /// </summary>
+        public bool PruneDisagreeingCopies { get; set; }
+
         /// <summary>
         /// Caps the concurrency used within a single parse or generation -- analysis cascade,
         /// affix-template unapplication and synthesis alike. A value of 1 runs the work fully

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs

@@ -29,7 +29,8 @@ public AnalysisAffixProcessRule(Morpher morpher, AffixProcessRule rule)
             foreach (AffixProcessAllomorph allo in rule.Allomorphs)
             {
                 _rules.Add(
-                    new MultiplePatternRule<Word, ShapeNode>(
+                    new CopyAgreementPatternRule(
+                        morpher,
                         new AnalysisAffixProcessAllomorphRuleSpec(allo),
                         new MatcherSettings<ShapeNode>
                         {

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisMorphologicalTransform.cs

@@ -46,6 +46,82 @@ protected IDictionary<string, int> CapturedParts
             get { return _capturedParts; }
         }
 
+        internal bool HasRepeatedParts
+        {
+            get { return _capturedParts.Values.Any(count => count >= 2); }
+        }
+
+        /// <summary>
+        /// Synthesis writes every copy of a part from the same input, and anything that later changes one
+        /// copy is unapplied before this rule, so copies proven to disagree segment by segment cannot lead
+        /// to a valid analysis. A copy containing an optional node (an unapplied deletion), or a part the
+        /// rule modifies, cannot be judged and never counts as disagreeing.
+        /// </summary>
+        internal bool HasDisagreeingCopies(Match<Word, ShapeNode> match)
+        {
+            foreach (KeyValuePair<string, int> capturedPart in _capturedParts)
+            {
+                string partName = capturedPart.Key;
+                int copyCount = capturedPart.Value;
+                if (copyCount < 2)
+                    continue;
+
+                if (_modifyFromInfos.ContainsKey(partName))
+                    continue;
+
+                var copies = new List<List<ShapeNode>>(copyCount);
+                bool partUndecidable = false;
+                for (int i = 0; i < copyCount; i++)
+                {
+                    GroupCapture<ShapeNode> capture = match.GroupCaptures[GetGroupName(partName, i)];
+                    if (!capture.Success)
+                    {
+                        partUndecidable = true;
+                        break;
+                    }
+
+                    List<ShapeNode> nodes = GetCapturedNodes(match.Input.Shape, capture.Range);
+                    if (nodes.Any(node => node.Annotation.Optional))
+                    {
+                        partUndecidable = true;
+                        break;
+                    }
+
+                    copies.Add(nodes.Where(node => node.Annotation.Type() == HCFeatureSystem.Segment).ToList());
+                }
+
+                if (partUndecidable)
+                    continue;
+
+                List<ShapeNode> firstCopy = copies[0];
+                for (int i = 1; i < copies.Count; i++)
+                {
+                    List<ShapeNode> otherCopy = copies[i];
+                    if (firstCopy.Count != otherCopy.Count)
+                        return true;
+
+                    for (int j = 0; j < firstCopy.Count; j++)
+                    {
+                        if (!firstCopy[j].Annotation.FeatureStruct.IsUnifiable(otherCopy[j].Annotation.FeatureStruct))
+                            return true;
+                    }
+                }
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Includes optional nodes the matcher skipped before the capture when they reach the start of the
+        /// shape, so an unapplied word-initial deletion is still seen.
+        /// </summary>
+        private static List<ShapeNode> GetCapturedNodes(Shape shape, Range<ShapeNode> range)
+        {
+            var nodes = new List<ShapeNode>(MorphologicalOutputAction.SkippedOptionalNodes(shape, range));
+            nodes.AddRange(shape.GetNodes(range));
+            return nodes;
+        }
+
         public Pattern<Word, ShapeNode> Pattern
         {
             get { return _pattern; }

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs

@@ -22,7 +22,8 @@ public AnalysisRealizationalAffixProcessRule(Morpher morpher, RealizationalAffix
             foreach (AffixProcessAllomorph allo in rule.Allomorphs)
             {
                 _rules.Add(
-                    new MultiplePatternRule<Word, ShapeNode>(
+                    new CopyAgreementPatternRule(
+                        morpher,
                         new AnalysisAffixProcessAllomorphRuleSpec(allo),
                         new MatcherSettings<ShapeNode>
                         {

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/CopyAgreementPatternRule.cs

@@ -0,0 +1,40 @@
+using System.Collections.Generic;
+using SIL.Machine.Annotations;
+using SIL.Machine.Matching;
+using SIL.Machine.Rules;
+
+namespace SIL.Machine.Morphology.HermitCrab.MorphologicalRules
+{
+    internal sealed class CopyAgreementPatternRule : MultiplePatternRule<Word, ShapeNode>
+    {
+        private readonly Morpher _morpher;
+        private readonly AnalysisAffixProcessAllomorphRuleSpec _spec;
+        private readonly bool _hasRepeatedParts;
+
+        public CopyAgreementPatternRule(
+            Morpher morpher,
+            AnalysisAffixProcessAllomorphRuleSpec ruleSpec,
+            MatcherSettings<ShapeNode> matcherSettings
+        )
+            : base(ruleSpec, matcherSettings)
+        {
+            _morpher = morpher;
+            _spec = ruleSpec;
+            _hasRepeatedParts = ruleSpec.HasRepeatedParts;
+        }
+
+        protected override IEnumerable<Word> ApplyImpl(Word input, ShapeNode start)
+        {
+            bool prune = _morpher.PruneDisagreeingCopies && _hasRepeatedParts;
+            var results = new List<Word>();
+            foreach (Match<Word, ShapeNode> match in Matcher.AllMatches(input, start))
+            {
+                if (prune && _spec.HasDisagreeingCopies(match))
+                    continue;
+
+                results.Add(RuleSpec.ApplyRhs(this, match));
+            }
+            return results;
+        }
+    }
+}

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/MorphologicalOutputAction.cs

@@ -38,6 +38,11 @@ IDictionary<string, int> capturedParts
         public abstract IEnumerable<Tuple<ShapeNode, ShapeNode>> Apply(Match<Word, ShapeNode> match, Word output);
 
         protected IEnumerable<ShapeNode> GetSkippedOptionalNodes(Shape shape, Range<ShapeNode> range)
+        {
+            return SkippedOptionalNodes(shape, range);
+        }
+
+        internal static IEnumerable<ShapeNode> SkippedOptionalNodes(Shape shape, Range<ShapeNode> range)
         {
             ShapeNode node = range.Start.Prev;
             var skippedNodes = new List<ShapeNode>();

tests/SIL.Machine.Morphology.HermitCrab.Tests/MorphologicalRules/AffixProcessRuleTests.cs

@@ -965,8 +965,9 @@ public void SimulfixRules()
         AssertMorphsEqual(morpher.ParseWord("sɯɯpu"), "50 SIMUL");
     }
 
-    [Test]
-    public void ReduplicationRules()
+    [TestCase(false)]
+    [TestCase(true)]
+    public void ReduplicationRules(bool pruneCopies)
     {
         var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value;
         var cons = FeatureStruct
@@ -1015,7 +1016,7 @@ public void ReduplicationRules()
         );
         Morphophonemic.MorphologicalRules.Add(redup);
 
-        var morpher = new Morpher(TraceManager, Language);
+        var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("sasag"), "RED 32");
 
         var voicing = new RewriteRule
@@ -1033,7 +1034,7 @@ public void ReduplicationRules()
         );
         Allophonic.PhonologicalRules.Add(voicing);
 
-        morpher = new Morpher(TraceManager, Language);
+        morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("sazag"), "RED 32");
 
         var affrication = new RewriteRule
@@ -1050,7 +1051,7 @@ public void ReduplicationRules()
         );
         Allophonic.PhonologicalRules.Add(affrication);
 
-        morpher = new Morpher(TraceManager, Language);
+        morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("tsazag"), "RED 32");
 
         redup.Allomorphs.Clear();
@@ -1066,7 +1067,7 @@ public void ReduplicationRules()
             }
         );
 
-        morpher = new Morpher(TraceManager, Language);
+        morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("tsagag"), "32 RED");
 
         Allophonic.PhonologicalRules.Clear();
@@ -1084,7 +1085,7 @@ public void ReduplicationRules()
             }
         );
 
-        morpher = new Morpher(TraceManager, Language);
+        morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("sagsag"), "32 RED");
         AssertMorphsEqual(morpher.ParseWord("sasibudbud"), "38 RED");
 
@@ -1115,7 +1116,7 @@ public void ReduplicationRules()
         );
         Allophonic.PhonologicalRules.Add(gDelete);
 
-        morpher = new Morpher(TraceManager, Language);
+        morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("saag"), "32 RED");
 
         gDelete.Subrules.Clear();
@@ -1126,7 +1127,7 @@ public void ReduplicationRules()
             }
         );
 
-        morpher = new Morpher(TraceManager, Language);
+        morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("saga"), "32 RED");
 
         Allophonic.PhonologicalRules.Clear();
@@ -1152,7 +1153,7 @@ public void ReduplicationRules()
             }
         );
 
-        morpher = new Morpher(TraceManager, Language);
+        morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("buiuib"), "30 RED", "31 RED");
     }
 
@@ -1900,8 +1901,9 @@ public void SubsumedAffix()
         AssertMorphsEqual(morpher.ParseWord("bubib"), "42 PRES", "43 PRES");
     }
 
-    [Test]
-    public void ModifyFromInputRules()
+    [TestCase(false)]
+    [TestCase(true)]
+    public void ModifyFromInputRules(bool pruneCopies)
     {
         var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value;
         var vowel = FeatureStruct
@@ -1941,7 +1943,7 @@ public void ModifyFromInputRules()
         );
         Morphophonemic.MorphologicalRules.Add(sSuffix);
 
-        var morpher = new Morpher(TraceManager, Language);
+        var morpher = new Morpher(TraceManager, Language) { PruneDisagreeingCopies = pruneCopies };
         AssertMorphsEqual(morpher.ParseWord("puso"), "52 PL");
     }