From fd517004b31558c6c7fd13967a030b421986a127 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Mon, 21 Sep 2026 07:40:23 -0700 Subject: [PATCH 1/2] Add UniqueStemMSAs --- Src/LexText/ParserCore/HCLoader.cs | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/Src/LexText/ParserCore/HCLoader.cs b/Src/LexText/ParserCore/HCLoader.cs index 4d2a889c92..ec2bc68794 100644 --- a/Src/LexText/ParserCore/HCLoader.cs +++ b/Src/LexText/ParserCore/HCLoader.cs @@ -636,7 +636,7 @@ protected virtual void LoadLexEntries(Stratum stratum, ILexEntry entry, IList()) + foreach (IMoStemMsa msa in UniqueStemMSAs(mainEntry.MorphoSyntaxAnalysesOC.OfType())) LoadLexEntryOfVariant(stratum, inflType, msa, allos, entry.ShortName); } else @@ -650,10 +650,32 @@ protected virtual void LoadLexEntries(Stratum stratum, ILexEntry entry, IList()) + foreach (IMoStemMsa msa in UniqueStemMSAs(entry.MorphoSyntaxAnalysesOC.OfType())) LoadLexEntry(stratum, msa, allos, entry.ShortName); } + IEnumerable UniqueStemMSAs(IEnumerable msas) + { + IList newMsas = new List(); + foreach (var msa in msas) + { + bool found = false; + foreach (var newMsa in newMsas) + { + if (newMsa.EqualsMsa(msa)) + { + found = true; + break; + } + } + if (!found) + { + newMsas.Add(msa); + } + } + return newMsas; + } + protected IEnumerable GetInflTypes(ILexEntryRef lexEntryRef) { if (lexEntryRef.VariantEntryTypesRS.Count == 0) From c6b7755e7e51e6b14cfb18f73704bdea599e0585 Mon Sep 17 00:00:00 2001 From: John Maxwell Date: Mon, 28 Sep 2026 08:33:23 -0700 Subject: [PATCH 2/2] Fixes bugs reported by John Lambert --- Src/LexText/ParserCore/HCLoader.cs | 72 ++++++++++++++++++++------- Src/LexText/ParserCore/HCParser.cs | 6 ++- Src/LexText/ParserCore/ParseResult.cs | 19 ++++++- 3 files changed, 75 insertions(+), 22 deletions(-) diff --git a/Src/LexText/ParserCore/HCLoader.cs b/Src/LexText/ParserCore/HCLoader.cs index ec2bc68794..13df43615b 100644 --- a/Src/LexText/ParserCore/HCLoader.cs +++ b/Src/LexText/ParserCore/HCLoader.cs @@ -34,6 +34,17 @@ public static Language Load(LcmCache cache, IHCLoadErrorLogger logger) return loader.m_language; } + public static Language Load( + LcmCache cache, + IHCLoadErrorLogger logger, + out Dictionary representativeMSAs) + { + var loader = new HCLoader(cache, logger); + loader.LoadLanguage(); + representativeMSAs = loader.m_representativeMSAs; + return loader.m_language; + } + private static readonly string[] VariableNames = { "α", "β", "γ", "δ", "ε", "ζ", "η", "θ", "ι", "κ", "λ", "μ", "ν", "ξ", @@ -43,6 +54,7 @@ public static Language Load(LcmCache cache, IHCLoadErrorLogger logger) private readonly LcmCache m_cache; protected readonly Dictionary> m_allomorphs; private readonly Dictionary> m_morphemes; + protected readonly Dictionary m_representativeMSAs; private readonly Dictionary m_stemNames; protected readonly Dictionary m_mprFeatures; @@ -78,6 +90,7 @@ protected HCLoader(LcmCache cache, IHCLoadErrorLogger logger) m_logger = logger; m_allomorphs = new Dictionary>(); m_morphemes = new Dictionary>(); + m_representativeMSAs = new Dictionary(); m_stemNames = new Dictionary(); m_mprFeatures = new Dictionary(); @@ -643,7 +656,12 @@ protected virtual void LoadLexEntries(Stratum stratum, ILexEntry entry, IList UniqueStemMSAs(IEnumerable msas) + IMoMorphSynAnalysis RepresentativeMSA(IMoMorphSynAnalysis msa) { - IList newMsas = new List(); - foreach (var msa in msas) + if (m_representativeMSAs.TryGetValue(msa, out IMoMorphSynAnalysis representativeMSA)) { - bool found = false; - foreach (var newMsa in newMsas) - { - if (newMsa.EqualsMsa(msa)) - { - found = true; - break; - } - } - if (!found) + return representativeMSA; + } + ILexEntry entry = msa.OwnerOfClass(); + foreach (var msa2 in entry.MorphoSyntaxAnalysesOC) + { + if (msa2.EqualsMsa(msa) && entry.SenseWithMsa(msa2) != null) { - newMsas.Add(msa); + m_representativeMSAs[msa] = msa2; + return msa2; } } + return msa; + } + + IEnumerable UniqueStemMSAs(IEnumerable msas) + { + HashSet newMsas = new HashSet(); + foreach (var msa in msas) + { + newMsas.Add((IMoStemMsa)RepresentativeMSA(msa)); + } + return newMsas; + } + + IEnumerable UniqueMSAs(IEnumerable msas) + { + HashSet newMsas = new HashSet(); + foreach (var msa in msas) + { + newMsas.Add(RepresentativeMSA(msa)); + } return newMsas; } @@ -880,19 +914,19 @@ protected virtual void LoadMorphologicalRules(Stratum stratum, ILexEntry entry, var mainEntry = component as ILexEntry; if (mainEntry != null) { - foreach (IMoMorphSynAnalysis msa in mainEntry.MorphoSyntaxAnalysesOC) + foreach (IMoMorphSynAnalysis msa in UniqueMSAs(mainEntry.MorphoSyntaxAnalysesOC)) LoadMorphologicalRule(stratum, entry, allos, msa); } else { var sense = (ILexSense)component; - LoadMorphologicalRule(stratum, entry, allos, sense.MorphoSyntaxAnalysisRA); + LoadMorphologicalRule(stratum, entry, allos, RepresentativeMSA(sense.MorphoSyntaxAnalysisRA)); } } } } - foreach (IMoMorphSynAnalysis msa in entry.MorphoSyntaxAnalysesOC) + foreach (IMoMorphSynAnalysis msa in UniqueMSAs(entry.MorphoSyntaxAnalysesOC)) LoadMorphologicalRule(stratum, entry, allos, msa); } @@ -2238,7 +2272,7 @@ private void LoadMorphemeCoOccurrenceRules(IMoMorphAdhocProhib morphAdhocProhib) if (m_morphemes.TryGetValue(morphAdhocProhib.FirstMorphemeRA, out firstMorphemes)) { var allOthers = new List>(); - foreach (IMoMorphSynAnalysis msa in morphAdhocProhib.RestOfMorphsRS) + foreach (IMoMorphSynAnalysis msa in UniqueMSAs(morphAdhocProhib.RestOfMorphsRS)) { List hcMorphemes; if (m_morphemes.TryGetValue(msa, out hcMorphemes)) diff --git a/Src/LexText/ParserCore/HCParser.cs b/Src/LexText/ParserCore/HCParser.cs index dd09d1c9ea..c2a3394a65 100644 --- a/Src/LexText/ParserCore/HCParser.cs +++ b/Src/LexText/ParserCore/HCParser.cs @@ -25,6 +25,7 @@ public class HCParser : DisposableBase, IParser private readonly LcmCache m_cache; private Morpher m_morpher; private Language m_language; + private Dictionary m_representativeMSAs; private readonly FwXmlTraceManager m_traceManager; private readonly string m_outputDirectory; private ParserModelChangeListener m_changeListener; @@ -109,7 +110,8 @@ public ParseResult ParseWord(string word) if (GetMorphs(wordAnalysis, out morphs)) { analyses.Add(new ParseAnalysis(morphs.Select(mi => - new ParseMorph(mi.Form, mi.Msa, mi.InflType, mi.GuessedString)))); + new ParseMorph(mi.Form, mi.Msa, mi.InflType, mi.GuessedString)), + m_representativeMSAs)); } } result = new ParseResult(analyses); @@ -156,7 +158,7 @@ private void LoadParser() using (new WorkerThreadReadHelper(m_cache.ServiceLocator.GetInstance())) { writer.WriteStartElement("LoadErrors"); - m_language = HCLoader.Load(m_cache, new XmlHCLoadErrorLogger(writer)); + m_language = HCLoader.Load(m_cache, new XmlHCLoadErrorLogger(writer), out m_representativeMSAs); writer.WriteEndElement(); XElement parserParamsElem = XElement.Parse(m_cache.LanguageProject.MorphologicalDataOA.ParserParameters); XElement delReappsElem = parserParamsElem.Elements("HC").Elements("DelReapps").FirstOrDefault(); diff --git a/Src/LexText/ParserCore/ParseResult.cs b/Src/LexText/ParserCore/ParseResult.cs index a29e287f31..77116350c5 100644 --- a/Src/LexText/ParserCore/ParseResult.cs +++ b/Src/LexText/ParserCore/ParseResult.cs @@ -72,12 +72,19 @@ public override int GetHashCode() public class ParseAnalysis : IEquatable { private readonly ReadOnlyCollection m_morphs; + private readonly Dictionary m_representativeMSAs; public ParseAnalysis(IEnumerable morphs) { m_morphs = new ReadOnlyCollection(morphs.ToArray()); } + public ParseAnalysis(IEnumerable morphs, Dictionary representativeMSAs) + { + m_morphs = new ReadOnlyCollection(morphs.ToArray()); + m_representativeMSAs = representativeMSAs; + } + public ReadOnlyCollection Morphs { get { return m_morphs; } @@ -115,7 +122,8 @@ public bool MatchesIWfiAnalysis(IWfiAnalysis analysis) foreach (IWfiMorphBundle mb in analysis.MorphBundlesOS) { var current = this.Morphs[i++]; - if (mb.MorphRA == current.Form && mb.MsaRA == current.Msa && mb.InflTypeRA == current.InflType && + var mbMSA = RepresentativeMSA(mb.MsaRA); + if (mb.MorphRA == current.Form && mbMSA == current.Msa && mb.InflTypeRA == current.InflType && (current.GuessedString == null || EquivalentFormString(mb.Form, current.GuessedString))) { // Possibly matches condition (2), above. @@ -132,6 +140,15 @@ public bool MatchesIWfiAnalysis(IWfiAnalysis analysis) return false; } + private IMoMorphSynAnalysis RepresentativeMSA(IMoMorphSynAnalysis msa) + { + if (msa != null && m_representativeMSAs != null && m_representativeMSAs.TryGetValue(msa, out IMoMorphSynAnalysis repMSA)) + { + return repMSA; + } + return msa; + } + private bool EquivalentFormString(IMultiString multiString, string formString) { foreach (int ws in multiString.AvailableWritingSystemIds)