diff --git a/Src/LexText/ParserCore/HCLoader.cs b/Src/LexText/ParserCore/HCLoader.cs index 4d2a889c92..035d2a5966 100644 --- a/Src/LexText/ParserCore/HCLoader.cs +++ b/Src/LexText/ParserCore/HCLoader.cs @@ -34,6 +34,17 @@ public static Language Load(LcmCache cache, IHCLoadErrorLogger logger) return loader.m_language; } + public static Language Load( + LcmCache cache, + IHCLoadErrorLogger logger, + out Dictionary representativeMSAs) + { + var loader = new HCLoader(cache, logger, true); + loader.LoadLanguage(); + representativeMSAs = loader.m_representativeMSAs; + return loader.m_language; + } + private static readonly string[] VariableNames = { "α", "β", "γ", "δ", "ε", "ζ", "η", "θ", "ι", "κ", "λ", "μ", "ν", "ξ", @@ -43,6 +54,7 @@ public static Language Load(LcmCache cache, IHCLoadErrorLogger logger) private readonly LcmCache m_cache; protected readonly Dictionary> m_allomorphs; private readonly Dictionary> m_morphemes; + protected readonly Dictionary m_representativeMSAs; private readonly Dictionary m_stemNames; protected readonly Dictionary m_mprFeatures; @@ -72,12 +84,13 @@ public static Language Load(LcmCache cache, IHCLoadErrorLogger logger) private CharacterDefinition m_morphBdry; protected static char m_ReplaceSpaceChar = '.'; - protected HCLoader(LcmCache cache, IHCLoadErrorLogger logger) + protected HCLoader(LcmCache cache, IHCLoadErrorLogger logger, bool mergeMSAs = false) { m_cache = cache; m_logger = logger; m_allomorphs = new Dictionary>(); m_morphemes = new Dictionary>(); + m_representativeMSAs = mergeMSAs ? new Dictionary() : null; m_stemNames = new Dictionary(); m_mprFeatures = new Dictionary(); @@ -636,24 +649,60 @@ protected virtual void LoadLexEntries(Stratum stratum, ILexEntry entry, IList()) + foreach (IMoStemMsa msa in UniqueMSAs(mainEntry.MorphoSyntaxAnalysesOC.OfType())) LoadLexEntryOfVariant(stratum, inflType, msa, allos, entry.ShortName); } else { ILexSense sense = (ILexSense)component; if (sense != null && sense.MorphoSyntaxAnalysisRA is IMoStemMsa) - LoadLexEntryOfVariant(stratum, inflType, (IMoStemMsa)sense.MorphoSyntaxAnalysisRA, allos, entry.ShortName); + LoadLexEntryOfVariant( + stratum, + inflType, + (IMoStemMsa)RepresentativeMSA(sense.MorphoSyntaxAnalysisRA), + allos, + entry.ShortName); } } } } } - foreach (IMoStemMsa msa in entry.MorphoSyntaxAnalysesOC.OfType()) + foreach (IMoStemMsa msa in UniqueMSAs(entry.MorphoSyntaxAnalysesOC.OfType())) LoadLexEntry(stratum, msa, allos, entry.ShortName); } + IMoMorphSynAnalysis RepresentativeMSA(IMoMorphSynAnalysis msa) + { + if (m_representativeMSAs == null) + { + return msa; + } + if (msa is IMoDerivStepMsa) + { + return msa; + } + if (m_representativeMSAs.TryGetValue(msa, out IMoMorphSynAnalysis representativeMSA)) + { + return representativeMSA; + } + ILexEntry entry = msa.OwnerOfClass(); + foreach (var msa2 in entry.MorphoSyntaxAnalysesOC) + { + if (msa2.ClassID == msa.ClassID && msa2.EqualsMsa(msa) && entry.SenseWithMsa(msa2) != null) + { + m_representativeMSAs[msa] = msa2; + return msa2; + } + } + return msa; + } + + private IEnumerable UniqueMSAs(IEnumerable msas) where T : class, IMoMorphSynAnalysis + { + return msas.Select(msa => (T)RepresentativeMSA(msa)).Distinct(); + } + protected IEnumerable GetInflTypes(ILexEntryRef lexEntryRef) { if (lexEntryRef.VariantEntryTypesRS.Count == 0) @@ -858,19 +907,19 @@ protected virtual void LoadMorphologicalRules(Stratum stratum, ILexEntry entry, var mainEntry = component as ILexEntry; if (mainEntry != null) { - foreach (IMoMorphSynAnalysis msa in mainEntry.MorphoSyntaxAnalysesOC) + foreach (IMoMorphSynAnalysis msa in UniqueMSAs(mainEntry.MorphoSyntaxAnalysesOC)) LoadMorphologicalRule(stratum, entry, allos, msa); } else { var sense = (ILexSense)component; - LoadMorphologicalRule(stratum, entry, allos, sense.MorphoSyntaxAnalysisRA); + LoadMorphologicalRule(stratum, entry, allos, RepresentativeMSA(sense.MorphoSyntaxAnalysisRA)); } } } } - foreach (IMoMorphSynAnalysis msa in entry.MorphoSyntaxAnalysesOC) + foreach (IMoMorphSynAnalysis msa in UniqueMSAs(entry.MorphoSyntaxAnalysesOC)) LoadMorphologicalRule(stratum, entry, allos, msa); } @@ -2213,10 +2262,10 @@ private IEnumerable Permute(List> items, int index) private void LoadMorphemeCoOccurrenceRules(IMoMorphAdhocProhib morphAdhocProhib) { List firstMorphemes; - if (m_morphemes.TryGetValue(morphAdhocProhib.FirstMorphemeRA, out firstMorphemes)) + if (m_morphemes.TryGetValue(RepresentativeMSA(morphAdhocProhib.FirstMorphemeRA), out firstMorphemes)) { var allOthers = new List>(); - foreach (IMoMorphSynAnalysis msa in morphAdhocProhib.RestOfMorphsRS) + foreach (IMoMorphSynAnalysis msa in UniqueMSAs(morphAdhocProhib.RestOfMorphsRS)) { List hcMorphemes; if (m_morphemes.TryGetValue(msa, out hcMorphemes)) diff --git a/Src/LexText/ParserCore/HCParser.cs b/Src/LexText/ParserCore/HCParser.cs index dd09d1c9ea..3462f37861 100644 --- a/Src/LexText/ParserCore/HCParser.cs +++ b/Src/LexText/ParserCore/HCParser.cs @@ -2,6 +2,14 @@ // This software is licensed under the LGPL, version 2.1 or later // (http://www.gnu.org/licenses/lgpl-2.1.html) +using SIL.Data; +using SIL.LCModel; +using SIL.LCModel.Infrastructure; +using SIL.Machine.Annotations; +using SIL.Machine.Morphology.HermitCrab; +using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; +using SIL.Machine.Rules; +using SIL.ObjectModel; using System; using System.Collections.Generic; using System.Globalization; @@ -10,13 +18,6 @@ using System.Text; using System.Xml; using System.Xml.Linq; -using SIL.LCModel; -using SIL.LCModel.Infrastructure; -using SIL.Machine.Annotations; -using SIL.Machine.Morphology.HermitCrab; -using SIL.Machine.Morphology.HermitCrab.MorphologicalRules; -using SIL.Machine.Rules; -using SIL.ObjectModel; namespace SIL.FieldWorks.WordWorks.Parser { @@ -25,12 +26,14 @@ public class HCParser : DisposableBase, IParser private readonly LcmCache m_cache; private Morpher m_morpher; private Language m_language; + private Dictionary m_representativeMSAs; private readonly FwXmlTraceManager m_traceManager; private readonly string m_outputDirectory; private ParserModelChangeListener m_changeListener; private bool m_forceUpdate; private bool m_guessRoots; private bool m_mergeAnalyses; + private bool m_mergeMSAs = true; // the public const strings are for GenerateHCConfigForFLExTrans and HCSynthByGlossLib internal const string CRuleID = "ID"; @@ -109,7 +112,8 @@ public ParseResult ParseWord(string word) if (GetMorphs(wordAnalysis, out morphs)) { analyses.Add(new ParseAnalysis(morphs.Select(mi => - new ParseMorph(mi.Form, mi.Msa, mi.InflType, mi.GuessedString)))); + new ParseMorph(mi.Form, mi.Msa, mi.InflType, mi.GuessedString)), + m_representativeMSAs)); } } result = new ParseResult(analyses); @@ -155,13 +159,11 @@ private void LoadParser() using (XmlWriter writer = XmlWriter.Create(loadErrorsFile)) using (new WorkerThreadReadHelper(m_cache.ServiceLocator.GetInstance())) { - writer.WriteStartElement("LoadErrors"); - m_language = HCLoader.Load(m_cache, new XmlHCLoadErrorLogger(writer)); - writer.WriteEndElement(); XElement parserParamsElem = XElement.Parse(m_cache.LanguageProject.MorphologicalDataOA.ParserParameters); XElement delReappsElem = parserParamsElem.Elements("HC").Elements("DelReapps").FirstOrDefault(); XElement guessRootsElem = parserParamsElem.Elements("HC").Elements("GuessRoots").FirstOrDefault(); XElement mergeAnalysesElem = parserParamsElem.Elements("HC").Elements("MergeAnalyses").FirstOrDefault(); + XElement mergeMSAsElem = parserParamsElem.Elements("HC").Elements("MergeMSAs").FirstOrDefault(); XElement maxRootsElem = parserParamsElem.Elements("HC").Elements("MaxRoots").FirstOrDefault(); XElement maxAlternativesElem = parserParamsElem.Elements("HC").Elements("MaxAlternatives").FirstOrDefault(); if (delReappsElem != null) @@ -169,11 +171,24 @@ private void LoadParser() if (guessRootsElem != null) m_guessRoots = (bool) guessRootsElem; if (mergeAnalysesElem != null) - m_mergeAnalyses = (bool) mergeAnalysesElem; + m_mergeAnalyses = (bool)mergeAnalysesElem; + if (mergeMSAsElem != null) + m_mergeMSAs = (bool)mergeMSAsElem; if (maxRootsElem != null) maxStemCount = int.Parse(maxRootsElem.Value); if (maxAlternativesElem != null) maxAlternatives = int.Parse(maxAlternativesElem.Value); + writer.WriteStartElement("LoadErrors"); + if (m_mergeMSAs) + { + m_language = HCLoader.Load(m_cache, new XmlHCLoadErrorLogger(writer), out m_representativeMSAs); + } + else + { + m_language = HCLoader.Load(m_cache, new XmlHCLoadErrorLogger(writer)); + m_representativeMSAs = null; + } + writer.WriteEndElement(); } m_morpher = new Morpher(m_traceManager, m_language) { DeletionReapplications = delReapps }; m_morpher.MaxStemCount = maxStemCount; @@ -191,7 +206,7 @@ private XDocument ParseToXml(string form, bool tracing, IEnumerable selectT { if (selectTraceMorphs != null) { - var selectTraceMorphsSet = new HashSet(selectTraceMorphs); + HashSet selectTraceMorphsSet = new HashSet(selectTraceMorphs.Select(hvo => RepresentativeMSAHvo(hvo)).Distinct()); m_morpher.LexEntrySelector = entry => selectTraceMorphsSet.Contains((int) entry.Properties[MsaID]); m_morpher.RuleSelector = rule => { @@ -233,6 +248,24 @@ private XDocument ParseToXml(string form, bool tracing, IEnumerable selectT return doc; } + private int RepresentativeMSAHvo(int hvo) + { + if (m_representativeMSAs == null) + { + return hvo; + } + IMoMorphSynAnalysisRepository repository = m_cache.LangProject.Services.GetInstance(); + if (repository.TryGetObject(hvo, out IMoMorphSynAnalysis msa) && m_representativeMSAs.ContainsKey(msa)) + { + return m_representativeMSAs[msa].Hvo; + } + else + { + return hvo; + } + + } + /// /// Check integrity of phoneme-based natural classes (PhNCSegments) /// when there are phonological features diff --git a/Src/LexText/ParserCore/ParseResult.cs b/Src/LexText/ParserCore/ParseResult.cs index a29e287f31..77116350c5 100644 --- a/Src/LexText/ParserCore/ParseResult.cs +++ b/Src/LexText/ParserCore/ParseResult.cs @@ -72,12 +72,19 @@ public override int GetHashCode() public class ParseAnalysis : IEquatable { private readonly ReadOnlyCollection m_morphs; + private readonly Dictionary m_representativeMSAs; public ParseAnalysis(IEnumerable morphs) { m_morphs = new ReadOnlyCollection(morphs.ToArray()); } + public ParseAnalysis(IEnumerable morphs, Dictionary representativeMSAs) + { + m_morphs = new ReadOnlyCollection(morphs.ToArray()); + m_representativeMSAs = representativeMSAs; + } + public ReadOnlyCollection Morphs { get { return m_morphs; } @@ -115,7 +122,8 @@ public bool MatchesIWfiAnalysis(IWfiAnalysis analysis) foreach (IWfiMorphBundle mb in analysis.MorphBundlesOS) { var current = this.Morphs[i++]; - if (mb.MorphRA == current.Form && mb.MsaRA == current.Msa && mb.InflTypeRA == current.InflType && + var mbMSA = RepresentativeMSA(mb.MsaRA); + if (mb.MorphRA == current.Form && mbMSA == current.Msa && mb.InflTypeRA == current.InflType && (current.GuessedString == null || EquivalentFormString(mb.Form, current.GuessedString))) { // Possibly matches condition (2), above. @@ -132,6 +140,15 @@ public bool MatchesIWfiAnalysis(IWfiAnalysis analysis) return false; } + private IMoMorphSynAnalysis RepresentativeMSA(IMoMorphSynAnalysis msa) + { + if (msa != null && m_representativeMSAs != null && m_representativeMSAs.TryGetValue(msa, out IMoMorphSynAnalysis repMSA)) + { + return repMSA; + } + return msa; + } + private bool EquivalentFormString(IMultiString multiString, string formString) { foreach (int ws in multiString.AvailableWritingSystemIds) diff --git a/Src/LexText/ParserCore/ParserCoreTests/HCLoaderTests.cs b/Src/LexText/ParserCore/ParserCoreTests/HCLoaderTests.cs index 45550323d2..bc63c44c5f 100644 --- a/Src/LexText/ParserCore/ParserCoreTests/HCLoaderTests.cs +++ b/Src/LexText/ParserCore/ParserCoreTests/HCLoaderTests.cs @@ -662,6 +662,42 @@ public void UnclassifedAffix() Assert.That(rule.IsPartial, Is.True); } + [Test] + public void RepresentativeMSAs() + { + ILexEntry lexEntry = AddEntry(MoMorphTypeTags.kguidMorphSuffix, "ɯd", "gloss", new SandboxGenericMSA { MsaType = MsaType.kUnclassified, MainPOS = m_verb }); + var senseFactory = Cache.ServiceLocator.GetInstance(); + var lexSense = senseFactory.Create(); + lexEntry.SensesOS.Add(lexSense); + IMoUnclassifiedAffixMsa msa = Cache.ServiceLocator.GetInstance().Create(); + lexEntry.MorphoSyntaxAnalysesOC.Add(msa); + lexSense.MorphoSyntaxAnalysisRA = msa; + msa.PartOfSpeechRA = m_verb; + + ILexEntry stemEntry = AddEntry(MoMorphTypeTags.kguidMorphStem, "a", "gloss", new SandboxGenericMSA { MsaType = MsaType.kStem }); + var stemSense = senseFactory.Create(); + stemEntry.SensesOS.Add(stemSense); + IMoStemMsa stemMsa = Cache.ServiceLocator.GetInstance().Create(); + stemEntry.MorphoSyntaxAnalysesOC.Add(stemMsa); + stemSense.MorphoSyntaxAnalysisRA = stemMsa; + + m_loadErrors.Clear(); + m_lang = HCLoader.Load(Cache, new TestHCLoadErrorLogger(m_loadErrors), out _); + + Assert.That(m_lang.Strata[0].Entries.Count, Is.EqualTo(1)); + Assert.That(m_lang.Strata[0].MorphologicalRules.Count, Is.EqualTo(1)); + var rule = (AffixProcessRule)m_lang.Strata[0].MorphologicalRules[0]; + + Assert.That(rule.RequiredSyntacticFeatureStruct.ToString(), Is.EqualTo("[POS:V]")); + Assert.That(rule.Gloss, Is.EqualTo("gloss")); + Assert.That(rule.IsPartial, Is.True); + + m_loadErrors.Clear(); + m_lang = HCLoader.Load(Cache, new TestHCLoadErrorLogger(m_loadErrors)); + Assert.That(m_lang.Strata[0].Entries.Count, Is.EqualTo(2)); + Assert.That(m_lang.Strata[0].MorphologicalRules.Count, Is.EqualTo(2)); + } + [Test] public void AffixNoMorphTypeSet() { diff --git a/Src/LexText/ParserUI/ParserParametersDlg.cs b/Src/LexText/ParserUI/ParserParametersDlg.cs index 5729da0ca0..2979af9a65 100644 --- a/Src/LexText/ParserUI/ParserParametersDlg.cs +++ b/Src/LexText/ParserUI/ParserParametersDlg.cs @@ -34,6 +34,7 @@ public class ParserParametersDlg : ParserParametersBase private const string AcceptUnspecifiedGraphemes = "AcceptUnspecifiedGraphemes"; private const string GuessRoots = "GuessRoots"; private const string MergeAnalyses = "MergeAnalyses"; + private const string MergeMSAs = "MergeMSAs"; private const string Strata = "Strata"; private const string XAmple = "XAmple"; @@ -282,8 +283,9 @@ public void SetDlgInfo(string title, string parserParameters, ILcmOwningSequence m_dataGrid2.TableStyles[0].GridColumnStyles[2].Width = 130; m_dataGrid2.TableStyles[0].GridColumnStyles[4].Width = 160; m_dataGrid2.TableStyles[0].GridColumnStyles[6].Width = 90; - m_dataGrid2.TableStyles[0].GridColumnStyles[7].Width = 100; - m_dataGrid2.TableStyles[0].GridColumnStyles[8].Width = 400; + m_dataGrid2.TableStyles[0].GridColumnStyles[7].Width = 80; + m_dataGrid2.TableStyles[0].GridColumnStyles[8].Width = 100; + m_dataGrid2.TableStyles[0].GridColumnStyles[9].Width = 400; m_compoundRules = compoundRules; if (m_compoundRules?.Count > 0) @@ -318,6 +320,8 @@ private void LoadParserData(DataSet dsParserParameters) hcElem.Add(new XElement(GuessRoots, true)); if (hcElem.Element(MergeAnalyses) == null) hcElem.Add(new XElement(MergeAnalyses, true)); + if (hcElem.Element(MergeMSAs) == null) + hcElem.Add(new XElement(MergeMSAs, true)); if (hcElem.Element(Strata) == null) hcElem.Add(new XElement(Strata, "")); @@ -372,6 +376,7 @@ private DataTable CreateHCDataTable() tblHC.Columns.Add(AcceptUnspecifiedGraphemes, typeof(bool)); tblHC.Columns.Add(GuessRoots, typeof(bool)); tblHC.Columns.Add(MergeAnalyses, typeof(bool)); + tblHC.Columns.Add(MergeMSAs, typeof(bool)); tblHC.Columns.Add(MaxAlternatives, typeof(int)); tblHC.Columns.Add(Strata, typeof(string)); return tblHC;