Skip to content

Add UniqueStemMSAs #1148

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
jtmaxwell3 wants to merge 2 commits into
base: main
Choose a base branch
from
+85 −10
Draft

Add UniqueStemMSAs #1148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fd51700
Add UniqueStemMSAs
jtmaxwell3 Sep 21, 2026
c6b7755
Fixes bugs reported by John Lambert
jtmaxwell3 Sep 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 63 additions & 7 deletions Src/LexText/ParserCore/HCLoader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,17 @@ public static Language Load(LcmCache cache, IHCLoadErrorLogger logger)
return loader.m_language;
}

public static Language Load(
LcmCache cache,
IHCLoadErrorLogger logger,
out Dictionary<IMoMorphSynAnalysis, IMoMorphSynAnalysis> representativeMSAs)
{
var loader = new HCLoader(cache, logger);
loader.LoadLanguage();
representativeMSAs = loader.m_representativeMSAs;
return loader.m_language;
}

private static readonly string[] VariableNames =
{
"α", "β", "γ", "δ", "ε", "ζ", "η", "θ", "ι", "κ", "λ", "μ", "ν", "ξ",
Expand All @@ -43,6 +54,7 @@ public static Language Load(LcmCache cache, IHCLoadErrorLogger logger)
private readonly LcmCache m_cache;
protected readonly Dictionary<IMoForm, List<Allomorph>> m_allomorphs;
private readonly Dictionary<IMoMorphSynAnalysis, List<Morpheme>> m_morphemes;
protected readonly Dictionary<IMoMorphSynAnalysis, IMoMorphSynAnalysis> m_representativeMSAs;
private readonly Dictionary<IMoStemName, StemName> m_stemNames;
protected readonly Dictionary<ICmObject, MprFeature> m_mprFeatures;

Expand Down Expand Up @@ -78,6 +90,7 @@ protected HCLoader(LcmCache cache, IHCLoadErrorLogger logger)
m_logger = logger;
m_allomorphs = new Dictionary<IMoForm, List<Allomorph>>();
m_morphemes = new Dictionary<IMoMorphSynAnalysis, List<Morpheme>>();
m_representativeMSAs = new Dictionary<IMoMorphSynAnalysis, IMoMorphSynAnalysis>();
m_stemNames = new Dictionary<IMoStemName, StemName>();
m_mprFeatures = new Dictionary<ICmObject, MprFeature>();

Expand Down Expand Up @@ -636,24 +649,67 @@ protected virtual void LoadLexEntries(Stratum stratum, ILexEntry entry, IList<IM
var mainEntry = component as ILexEntry;
if (mainEntry != null)
{
foreach (IMoStemMsa msa in mainEntry.MorphoSyntaxAnalysesOC.OfType<IMoStemMsa>())
foreach (IMoStemMsa msa in UniqueStemMSAs(mainEntry.MorphoSyntaxAnalysesOC.OfType<IMoStemMsa>()))
LoadLexEntryOfVariant(stratum, inflType, msa, allos, entry.ShortName);
}
else
{
ILexSense sense = (ILexSense)component;
if (sense != null && sense.MorphoSyntaxAnalysisRA is IMoStemMsa)
LoadLexEntryOfVariant(stratum, inflType, (IMoStemMsa)sense.MorphoSyntaxAnalysisRA, allos, entry.ShortName);
LoadLexEntryOfVariant(
stratum,
inflType,
(IMoStemMsa)RepresentativeMSA(sense.MorphoSyntaxAnalysisRA),
allos,
entry.ShortName);
}
}
}
}
}

foreach (IMoStemMsa msa in entry.MorphoSyntaxAnalysesOC.OfType<IMoStemMsa>())
foreach (IMoStemMsa msa in UniqueStemMSAs(entry.MorphoSyntaxAnalysesOC.OfType<IMoStemMsa>()))
LoadLexEntry(stratum, msa, allos, entry.ShortName);
}

IMoMorphSynAnalysis RepresentativeMSA(IMoMorphSynAnalysis msa)
{
if (m_representativeMSAs.TryGetValue(msa, out IMoMorphSynAnalysis representativeMSA))
{
return representativeMSA;
}
ILexEntry entry = msa.OwnerOfClass<ILexEntry>();
foreach (var msa2 in entry.MorphoSyntaxAnalysesOC)
{
if (msa2.EqualsMsa(msa) && entry.SenseWithMsa(msa2) != null)
{
m_representativeMSAs[msa] = msa2;
return msa2;
}
}
return msa;
}

IEnumerable<IMoStemMsa> UniqueStemMSAs(IEnumerable<IMoStemMsa> msas)
{
HashSet<IMoStemMsa> newMsas = new HashSet<IMoStemMsa>();
foreach (var msa in msas)
{
newMsas.Add((IMoStemMsa)RepresentativeMSA(msa));
}
return newMsas;
}

IEnumerable<IMoMorphSynAnalysis> UniqueMSAs(IEnumerable<IMoMorphSynAnalysis> msas)
{
HashSet<IMoMorphSynAnalysis> newMsas = new HashSet<IMoMorphSynAnalysis>();
foreach (var msa in msas)
{
newMsas.Add(RepresentativeMSA(msa));
}
return newMsas;
}

protected IEnumerable<ILexEntryInflType> GetInflTypes(ILexEntryRef lexEntryRef)
{
if (lexEntryRef.VariantEntryTypesRS.Count == 0)
Expand Down Expand Up @@ -858,19 +914,19 @@ protected virtual void LoadMorphologicalRules(Stratum stratum, ILexEntry entry,
var mainEntry = component as ILexEntry;
if (mainEntry != null)
{
foreach (IMoMorphSynAnalysis msa in mainEntry.MorphoSyntaxAnalysesOC)
foreach (IMoMorphSynAnalysis msa in UniqueMSAs(mainEntry.MorphoSyntaxAnalysesOC))
LoadMorphologicalRule(stratum, entry, allos, msa);
}
else
{
var sense = (ILexSense)component;
LoadMorphologicalRule(stratum, entry, allos, sense.MorphoSyntaxAnalysisRA);
LoadMorphologicalRule(stratum, entry, allos, RepresentativeMSA(sense.MorphoSyntaxAnalysisRA));
}
}
}
}

foreach (IMoMorphSynAnalysis msa in entry.MorphoSyntaxAnalysesOC)
foreach (IMoMorphSynAnalysis msa in UniqueMSAs(entry.MorphoSyntaxAnalysesOC))
LoadMorphologicalRule(stratum, entry, allos, msa);
}

Expand Down Expand Up @@ -2216,7 +2272,7 @@ private void LoadMorphemeCoOccurrenceRules(IMoMorphAdhocProhib morphAdhocProhib)
if (m_morphemes.TryGetValue(morphAdhocProhib.FirstMorphemeRA, out firstMorphemes))
{
var allOthers = new List<List<Morpheme>>();
foreach (IMoMorphSynAnalysis msa in morphAdhocProhib.RestOfMorphsRS)
foreach (IMoMorphSynAnalysis msa in UniqueMSAs(morphAdhocProhib.RestOfMorphsRS))
{
List<Morpheme> hcMorphemes;
if (m_morphemes.TryGetValue(msa, out hcMorphemes))
Expand Down
6 changes: 4 additions & 2 deletions Src/LexText/ParserCore/HCParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ public class HCParser : DisposableBase, IParser
private readonly LcmCache m_cache;
private Morpher m_morpher;
private Language m_language;
private Dictionary<IMoMorphSynAnalysis, IMoMorphSynAnalysis> m_representativeMSAs;
private readonly FwXmlTraceManager m_traceManager;
private readonly string m_outputDirectory;
private ParserModelChangeListener m_changeListener;
Expand Down Expand Up @@ -109,7 +110,8 @@ public ParseResult ParseWord(string word)
if (GetMorphs(wordAnalysis, out morphs))
{
analyses.Add(new ParseAnalysis(morphs.Select(mi =>
new ParseMorph(mi.Form, mi.Msa, mi.InflType, mi.GuessedString))));
new ParseMorph(mi.Form, mi.Msa, mi.InflType, mi.GuessedString)),
m_representativeMSAs));
}
}
result = new ParseResult(analyses);
Expand Down Expand Up @@ -156,7 +158,7 @@ private void LoadParser()
using (new WorkerThreadReadHelper(m_cache.ServiceLocator.GetInstance<IWorkerThreadReadHandler>()))
{
writer.WriteStartElement("LoadErrors");
m_language = HCLoader.Load(m_cache, new XmlHCLoadErrorLogger(writer));
m_language = HCLoader.Load(m_cache, new XmlHCLoadErrorLogger(writer), out m_representativeMSAs);
writer.WriteEndElement();
XElement parserParamsElem = XElement.Parse(m_cache.LanguageProject.MorphologicalDataOA.ParserParameters);
XElement delReappsElem = parserParamsElem.Elements("HC").Elements("DelReapps").FirstOrDefault();
Expand Down
19 changes: 18 additions & 1 deletion Src/LexText/ParserCore/ParseResult.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,19 @@ public override int GetHashCode()
public class ParseAnalysis : IEquatable<ParseAnalysis>
{
private readonly ReadOnlyCollection<ParseMorph> m_morphs;
private readonly Dictionary<IMoMorphSynAnalysis, IMoMorphSynAnalysis> m_representativeMSAs;

public ParseAnalysis(IEnumerable<ParseMorph> morphs)
{
m_morphs = new ReadOnlyCollection<ParseMorph>(morphs.ToArray());
}

public ParseAnalysis(IEnumerable<ParseMorph> morphs, Dictionary<IMoMorphSynAnalysis, IMoMorphSynAnalysis> representativeMSAs)
{
m_morphs = new ReadOnlyCollection<ParseMorph>(morphs.ToArray());
m_representativeMSAs = representativeMSAs;
}

public ReadOnlyCollection<ParseMorph> Morphs
{
get { return m_morphs; }
Expand Down Expand Up @@ -115,7 +122,8 @@ public bool MatchesIWfiAnalysis(IWfiAnalysis analysis)
foreach (IWfiMorphBundle mb in analysis.MorphBundlesOS)
{
var current = this.Morphs[i++];
if (mb.MorphRA == current.Form && mb.MsaRA == current.Msa && mb.InflTypeRA == current.InflType &&
var mbMSA = RepresentativeMSA(mb.MsaRA);
if (mb.MorphRA == current.Form && mbMSA == current.Msa && mb.InflTypeRA == current.InflType &&
(current.GuessedString == null || EquivalentFormString(mb.Form, current.GuessedString)))
{
// Possibly matches condition (2), above.
Expand All @@ -132,6 +140,15 @@ public bool MatchesIWfiAnalysis(IWfiAnalysis analysis)
return false;
}

private IMoMorphSynAnalysis RepresentativeMSA(IMoMorphSynAnalysis msa)
{
if (msa != null && m_representativeMSAs != null && m_representativeMSAs.TryGetValue(msa, out IMoMorphSynAnalysis repMSA))
{
return repMSA;
}
return msa;
}

private bool EquivalentFormString(IMultiString multiString, string formString)
{
foreach (int ws in multiString.AvailableWritingSystemIds)
Expand Down
Loading