From ebdc736bb39ded37a258101f998b2144fef9dc82 Mon Sep 17 00:00:00 2001 From: John T Maxwell III Date: Wed, 31 Jul 2024 13:59:05 -0700 Subject: [PATCH] LT-21586: Export/import Phonology (#307) * Add code to import and export phonology * Make ImportData accept M3Dump format * Handle natural class segments, add TestProject * Replace UpdateNCSegments with a more general mechanism for links * Include version and default vernacular writing system --- .../AppStrings.Designer.cs | 20 +- .../ApplicationServices/AppStrings.resx | 6 + .../ApplicationServices/XmlImportData.cs | 178 ++- .../DomainServices/M3ModelExportServices.cs | 238 +++- .../DomainServices/PhonologyServices.cs | 115 ++ .../DomainServices/PhonologyServicesTest.cs | 1193 +++++++++++++++++ 6 files changed, 1664 insertions(+), 86 deletions(-) create mode 100644 src/SIL.LCModel/DomainServices/PhonologyServices.cs create mode 100644 tests/SIL.LCModel.Tests/DomainServices/PhonologyServicesTest.cs diff --git a/src/SIL.LCModel/Application/ApplicationServices/AppStrings.Designer.cs b/src/SIL.LCModel/Application/ApplicationServices/AppStrings.Designer.cs index 7334e5d4..1f9f6fba 100644 --- a/src/SIL.LCModel/Application/ApplicationServices/AppStrings.Designer.cs +++ b/src/SIL.LCModel/Application/ApplicationServices/AppStrings.Designer.cs @@ -19,7 +19,7 @@ namespace SIL.LCModel.Application.ApplicationServices { // class via a tool like ResGen or Visual Studio. // To add or remove a member, edit your .ResX file then rerun ResGen // with the /str option, or rebuild your VS project. - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "16.0.0.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "17.0.0.0")] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] internal class AppStrings { @@ -637,5 +637,23 @@ internal static string ksUnrecognizedOwnerlessObjectClass { return ResourceManager.GetString("ksUnrecognizedOwnerlessObjectClass", resourceCulture); } } + + /// + /// Looks up a localized string similar to Error: {0} has the wrong default vernacular writing system (expecting {1}, was {2}). Import aborted.. + /// + internal static string ksWrongVernWs { + get { + return ResourceManager.GetString("ksWrongVernWs", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Error: {0} has an invalid version number (expecting {1}, was {2}). Import aborted.. + /// + internal static string ksWrongVersion { + get { + return ResourceManager.GetString("ksWrongVersion", resourceCulture); + } + } } } diff --git a/src/SIL.LCModel/Application/ApplicationServices/AppStrings.resx b/src/SIL.LCModel/Application/ApplicationServices/AppStrings.resx index fbd769c1..2c796e4a 100644 --- a/src/SIL.LCModel/Application/ApplicationServices/AppStrings.resx +++ b/src/SIL.LCModel/Application/ApplicationServices/AppStrings.resx @@ -318,4 +318,10 @@ {0}: Could not create Show Subentry Under link to entry "{1}", because it does not exist. Shown during import. The current entry is supposed to be shown as a subentry under "{1}". + + Error: {0} has the wrong default vernacular writing system (expecting {1}, was {2}). Import aborted. + + + Error: {0} has an invalid version number (expecting {1}, was {2}). Import aborted. + \ No newline at end of file diff --git a/src/SIL.LCModel/Application/ApplicationServices/XmlImportData.cs b/src/SIL.LCModel/Application/ApplicationServices/XmlImportData.cs index 95b01ee9..6b265d47 100644 --- a/src/SIL.LCModel/Application/ApplicationServices/XmlImportData.cs +++ b/src/SIL.LCModel/Application/ApplicationServices/XmlImportData.cs @@ -7,8 +7,10 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Security.Cryptography; using System.Text; using System.Xml; +using System.Xml.Linq; using Icu; using SIL.LCModel.Core.Cellar; using SIL.LCModel.Core.KernelInterfaces; @@ -18,6 +20,7 @@ using SIL.LCModel.DomainServices; using SIL.LCModel.Infrastructure; using SIL.LCModel.Utils; +using static SIL.LCModel.Application.ApplicationServices.XmlImportData; namespace SIL.LCModel.Application.ApplicationServices { @@ -92,7 +95,7 @@ internal class PendingLink string m_sState; Dictionary m_dictAttrs = new Dictionary(); - internal PendingLink(FieldInfo fi, XmlReader xrdr) + internal PendingLink(FieldInfo fi, XmlReader xrdr, string linkAttribute) { m_fi = fi; m_sName = xrdr.Name; @@ -101,7 +104,13 @@ internal PendingLink(FieldInfo fi, XmlReader xrdr) m_line = (xrdr as IXmlLineInfo).LineNumber; else m_line = 0; - if (xrdr.MoveToFirstAttribute()) + if (linkAttribute != null) + { + // The Phonology format sometimes uses attributes instead of fields in the XML. + // Save the value under "dst" and don't move the attribute. + m_dictAttrs.Add("dst", xrdr.GetAttribute(linkAttribute)); + } + else if (xrdr.MoveToFirstAttribute()) { do { @@ -150,6 +159,7 @@ internal string XmlState private Dictionary m_mapIdGuid = new Dictionary(); private Dictionary m_mapGuidId = new Dictionary(); + bool m_phonology = false; private ReferenceTracker m_rglinks = new ReferenceTracker(); private TextWriter m_wrtrLog; @@ -273,6 +283,26 @@ public void ImportData(TextReader rdr, TextWriter wrtrLog, IProgress progress) xrdr.Read(); xrdr.MoveToContent(); } + if (xrdr.Name == "Phonology") + { + // We are reading data in the Phonology format. + m_phonology = true; + string versionId = xrdr.GetAttribute("Version"); + if (versionId != null && versionId != PhonologyServices.VersionId) + { + string sMsg = String.Format(AppStrings.ksWrongVersion, m_sFilename, PhonologyServices.VersionId, versionId); + throw new Exception(sMsg); + } + string vernWs = xrdr.GetAttribute("DefaultVernWs"); + string cacheVernWs = m_cache.ServiceLocator.WritingSystems.DefaultVernacularWritingSystem.IcuLocale; + if (vernWs != null && vernWs != cacheVernWs) + { + string sMsg = String.Format(AppStrings.ksWrongVernWs, m_sFilename, cacheVernWs, vernWs); + throw new Exception(sMsg); + } + xrdr.Read(); + xrdr.MoveToContent(); + } int nOuterObjLevel = xrdr.Depth; while (!xrdr.EOF && xrdr.Depth >= nOuterObjLevel) { @@ -942,11 +972,15 @@ private void ReadXmlObject(XmlReader xrdr, FieldInfo fi, ICmObject objToUse) { Debug.Assert(xrdr.NodeType == XmlNodeType.Element); + string sClass = xrdr.Name; #if DEBUG int nDepth = xrdr.Depth; + string sOriginalClass = sClass; #endif - string sClass = xrdr.Name; - string sId = xrdr.GetAttribute("id"); + if (sClass == "PhonRuleFeat") + sClass = "PhPhonRuleFeat"; + // The Phonology format uses "Id" instead of "id". + string sId = xrdr.GetAttribute(m_phonology ? "Id" : "id"); ICmObject cmo = null; // Check for singleton classes that should already exist before creating new // objects. @@ -961,6 +995,14 @@ private void ReadXmlObject(XmlReader xrdr, FieldInfo fi, ICmObject objToUse) cmo = m_cache.LangProject.LexDbOA; Debug.Assert(cmo != null); break; + case "PhPhonData": + cmo = m_cache.LangProject.PhonologicalDataOA; + Debug.Assert(cmo != null); + break; + case "PhFeatureSystem": + cmo = m_cache.LangProject.PhFeatureSystemOA; + Debug.Assert(cmo != null); + break; default: int clid = m_mdc.GetClassId(sClass); if (fi != null) @@ -1004,6 +1046,11 @@ private void ReadXmlObject(XmlReader xrdr, FieldInfo fi, ICmObject objToUse) if (m_repoCmObject == null) m_repoCmObject = m_cache.ServiceLocator.GetInstance(); cmo = m_repoCmObject.GetObject(hvo); + // Remove the default code added by PhTerminalUnit.SetDefaultValuesAfterInit in OverridesLing_Lex. + (cmo as PhPhoneme)?.CodesOS.Clear(); + (cmo as PhBdryMarker)?.CodesOS.Clear(); + // Remove default values. + (cmo as PhRegularRule)?.RightHandSidesOS.Clear(); } else { @@ -1051,11 +1098,17 @@ private void ReadXmlObject(XmlReader xrdr, FieldInfo fi, ICmObject objToUse) m_mapIdGuid.Add(sId, cmo.Guid); m_mapGuidId.Add(cmo.Guid, sId); } + if (m_phonology) + // The Phonology format sometimes uses attributes instead of fields in the XML. + ReadXmlAttributes(xrdr, cmo, fNewObject, fi); if (!xrdr.IsEmptyElement) { - ReadXmlFields(xrdr, cmo, fNewObject, fi); + if (sClass == "FsFeatStruc") + ReadFeatureStructure(xrdr, cmo, fNewObject, fi); + else + ReadXmlFields(xrdr, cmo, fNewObject, fi); #if DEBUG - Debug.Assert(xrdr.Name == sClass); // we should be on the end element + Debug.Assert(xrdr.Name == sOriginalClass); // we should be on the end element Debug.Assert(xrdr.Depth == nDepth); #endif xrdr.ReadEndElement(); @@ -1094,7 +1147,62 @@ private void ReadXmlObject(XmlReader xrdr, FieldInfo fi, ICmObject objToUse) } } - private int LineNumber(XmlReader xrdr) + /// + /// Read the XML attributes without moving the XmlReader. + /// + private void ReadXmlAttributes(XmlReader xrdr, ICmObject cmo, bool fNewObject, FieldInfo fi) + { + IEnumerable attributeNames = new List() + { + "Direction", "Disabled", "dst", + "Feature", "Guid", "Id", + "LeftContext", "Maximum", "Minimum", + "RightContext", "Type", "Value" + }; + int attributeCount = 0; + foreach (string attributeName in attributeNames) + { + if (xrdr.GetAttribute(attributeName) != null) + { + attributeCount++; + if (attributeName == "Id" || attributeName == "Guid") + continue; + string fieldName = attributeName; + if (attributeName == "dst" && + ((xrdr.Name == "PhSimpleContextBdry" || + xrdr.Name == "PhSimpleContextNC" || + xrdr.Name == "PhSimpleContextSeg"))) + { + fieldName = "FeatureStructure"; + } + var flid = m_mdc.GetFieldId2(cmo.ClassID, fieldName, true); + var cpt = (CellarPropertyType)m_mdc.GetFieldType(flid); + string sVal = xrdr.GetAttribute(attributeName); + switch (cpt) + { + case CellarPropertyType.Boolean: + sVal = sVal.ToLowerInvariant(); + bool fVal = sVal == "true" || sVal == "yes" || sVal == "t" || sVal == "y" || sVal == "1"; + m_sda.SetBoolean(cmo.Hvo, flid, fVal); + break; + case CellarPropertyType.Integer: + int iVal = Int32.Parse(sVal); + m_sda.SetInt(cmo.Hvo, flid, iVal); + break; + case CellarPropertyType.ReferenceAtom: + var fsfi = new FieldInfo(cmo, flid, cpt, fNewObject, fi); + ReadReferenceLink(xrdr, fsfi, attributeName); + break; + default: + Debug.Assert(false); + break; + } + } + } + Debug.Assert(xrdr.AttributeCount == attributeCount); + } + + private int LineNumber(XmlReader xrdr) { if (xrdr != null) { @@ -1108,6 +1216,22 @@ private int LineNumber(XmlReader xrdr) const int kflidCrossReferences = -123; const int kflidLexicalRelations = -124; + private void ReadFeatureStructure(XmlReader xrdr, ICmObject cmoOwner, bool fOwnerIsNew, + FieldInfo fiParent) + { + int nDepth = xrdr.Depth; + // Consume the start element of the owning object. + xrdr.Read(); + xrdr.MoveToContent(); + var flid = m_mdc.GetFieldId2(cmoOwner.ClassID, "FeatureSpecs", true); + var cpt = (CellarPropertyType)m_mdc.GetFieldType(flid); + var fi = new FieldInfo(cmoOwner, flid, cpt, fOwnerIsNew, fiParent); + while (xrdr.Depth > nDepth) + { + ReadXmlObject(xrdr, fi, null); + } + } + private void ReadXmlFields(XmlReader xrdr, ICmObject cmoOwner, bool fOwnerIsNew, FieldInfo fiParent) { @@ -1117,8 +1241,9 @@ private void ReadXmlFields(XmlReader xrdr, ICmObject cmoOwner, bool fOwnerIsNew, xrdr.MoveToContent(); while (xrdr.Depth > nDepth) { - while (xrdr.IsEmptyElement) + while (xrdr.IsEmptyElement && xrdr.GetAttribute("dst") == null) { + // The Phonology format uses a "dst" attribute on an empty element to represent a link. xrdr.Read(); xrdr.MoveToContent(); } @@ -1131,6 +1256,10 @@ private void ReadXmlFields(XmlReader xrdr, ICmObject cmoOwner, bool fOwnerIsNew, CellarPropertyType cpt; ICmObject realOwner = null; FieldInfo fi = null; + if (sField == "PhonologicalFeatures" && cmoOwner.ClassName == "PhPhoneme") + sField = "Features"; + else if (sField == "FeatureConstraints" && cmoOwner.ClassName == "PhPhonData") + sField = "FeatConstraints"; if (cmoOwner.ClassID == LexDbTags.kClassId && sField == "Entries") { flid = 0; // no actual owning sequence. @@ -1194,8 +1323,12 @@ private void ReadXmlFields(XmlReader xrdr, ICmObject cmoOwner, bool fOwnerIsNew, } if (realOwner == null) fi = new FieldInfo(cmoOwner, flid, cpt, fOwnerIsNew, fiParent); - xrdr.Read(); - xrdr.MoveToContent(); + bool isEmptyDst = xrdr.GetAttribute("dst") != null && xrdr.IsEmptyElement; + if (xrdr.GetAttribute("dst") == null) + { + xrdr.Read(); + xrdr.MoveToContent(); + } if (xrdr.NodeType == XmlNodeType.EndElement && xrdr.Name == sField && xrdr.Depth == nDepthField) { // On Linux/Mono, empty elements can end up with both a start element node @@ -1291,6 +1424,8 @@ private void ReadXmlFields(XmlReader xrdr, ICmObject cmoOwner, bool fOwnerIsNew, } while (xrdr.Depth > nDepthField); break; } + if (isEmptyDst) + continue; if (xrdr.Depth > nDepth) { while (xrdr.IsStartElement()) @@ -1668,7 +1803,7 @@ private void EnsureAllWritingSystemsDefined(string sXml) } } - private void ReadReferenceLink(XmlReader xrdr, FieldInfo fi) + private void ReadReferenceLink(XmlReader xrdr, FieldInfo fi, string linkAttribute = null) { // This is going to be difficult, because all sorts of variants of links exist. // The simplest has a target attribute which refers to the id attribute of another @@ -1677,13 +1812,17 @@ private void ReadReferenceLink(XmlReader xrdr, FieldInfo fi) // input value along with all the attributes. Later, after everything has been // imported, so that all references can be resolved to actual objects, we'll try // to decipher all the pending reference links. - if (xrdr.Name != "Link") + // NB: The Phonology format uses a "dst" attribute instead of a "Link" element for links. + if (xrdr.Name != "Link" && xrdr.GetAttribute("dst") == null && linkAttribute == null) { string sMsg = AppStrings.ksExpectedLink; LogMessage(sMsg, LineNumber(xrdr)); throw new Exception(sMsg); } - PendingLink pend = new PendingLink(fi, xrdr); + if (linkAttribute != null && xrdr.GetAttribute(linkAttribute) == "0") + // Link is empty. + return; + PendingLink pend = new PendingLink(fi, xrdr, linkAttribute); if (pend.LinkAttributes.Count == 0) { string sMsg = AppStrings.ksInvalidLinkElement; @@ -1737,6 +1876,8 @@ private void ReadReferenceLink(XmlReader xrdr, FieldInfo fi) { m_rglinks.Add(pend); } + if (linkAttribute != null) + return; xrdr.MoveToElement(); xrdr.Read(); xrdr.MoveToContent(); @@ -2203,6 +2344,10 @@ private void FixPendingLinks() //This is a reversal index and we need a back reference set rie.SensesRS.Add(pend.FieldInformation.Owner as ILexSense); } + else if (pend.ElementName == "FeatureConstraints" && pend.FieldInformation.Owner is PhRegularRule) + { + // This is a synthetic feature. We can't set it, so we ignore it. + } else { // Some normal reference collection, just add the reference @@ -2995,6 +3140,13 @@ private int ResolveLinkReference(int flid, PendingLink pend, bool fHandleForm) return GetPictureFile(sPath, pend); } } + string sDst; + if (dictAttrs.TryGetValue("dst", out sDst)) + { + Guid guid = m_mapIdGuid[sDst]; + ICmObject cmo = m_repoCmObject.GetObject(guid); + return cmo.Hvo; + } return 0; } diff --git a/src/SIL.LCModel/DomainServices/M3ModelExportServices.cs b/src/SIL.LCModel/DomainServices/M3ModelExportServices.cs index 3074ede6..0ce09b8c 100644 --- a/src/SIL.LCModel/DomainServices/M3ModelExportServices.cs +++ b/src/SIL.LCModel/DomainServices/M3ModelExportServices.cs @@ -4,11 +4,17 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Text; using System.Xml.Linq; using Icu; using SIL.LCModel.Core.KernelInterfaces; +using SIL.LCModel.Core.Text; using SIL.LCModel.Core.WritingSystems; +using SIL.LCModel.DomainImpl; +using SIL.LCModel.Infrastructure; +using static SIL.LCModel.Application.ApplicationServices.XmlImportData; namespace SIL.LCModel.DomainServices { @@ -81,6 +87,29 @@ public static XDocument ExportGrammarAndLexicon(ILangProject languageProject) return doc; } + /// + /// Export the phonology of languageProject in the Phonology format. + /// + /// + /// + public static XDocument ExportPhonology(ILangProject languageProject) + { + if (languageProject == null) throw new ArgumentNullException("languageProject"); + string vernWs = languageProject.Cache.ServiceLocator.WritingSystems.DefaultVernacularWritingSystem.IcuLocale; + + const Normalizer.UNormalizationMode mode = Normalizer.UNormalizationMode.UNORM_NFD; + var doc = new XDocument( + new XDeclaration("1.0", "utf-8", "yes"), + new XElement("Phonology", + new XAttribute("Version", PhonologyServices.VersionId), + new XAttribute("DefaultVernWs", vernWs), + ExportPhonologicalData(languageProject.PhonologicalDataOA, mode, useMultiStrings: true, phonology: true), + ExportFeatureSystem(languageProject.PhFeatureSystemOA, "PhFeatureSystem", mode, useMultiStrings: true) + ) + ); + return doc; + } + private static XElement ExportLanguageProject(ILangProject languageProject, Normalizer.UNormalizationMode mode) { return new XElement("LangProject", @@ -453,32 +482,26 @@ select ExportRuleMapping(mapping))) // ExportMorphTypes rules go above this line. - private static XElement ExportPhonologicalData(IPhPhonData phonologicalData, Normalizer.UNormalizationMode mode) + private static XElement ExportPhonologicalData(IPhPhonData phonologicalData, Normalizer.UNormalizationMode mode, bool useMultiStrings = false, bool phonology = false) { return new XElement("PhPhonData", new XAttribute("Id", phonologicalData.Hvo), new XElement("Environments", - from goodEnvironment in phonologicalData.Services.GetInstance().AllValidInstances() - select new XElement("PhEnvironment", - new XAttribute("Id", goodEnvironment.Hvo), - new XAttribute("StringRepresentation", - Normalize(goodEnvironment.StringRepresentation, mode)), - CreateAttribute("LeftContext", goodEnvironment.LeftContextRA), - CreateAttribute("RightContext", goodEnvironment.RightContextRA), - ExportBestAnalysis(goodEnvironment.Name, "Name", mode), - ExportBestAnalysis(goodEnvironment.Description, "Description", mode))), + from goodEnvironment in GetPhEnvironments(phonologicalData) + select ExportPhEnvironment(goodEnvironment, mode, useMultiStrings)), new XElement("NaturalClasses", from naturalClass in phonologicalData.NaturalClassesOS - select ExportNaturalClass(naturalClass, mode)), + select ExportNaturalClass(naturalClass, mode, useMultiStrings)), new XElement("Contexts", from context in phonologicalData.ContextsOS select ExportContext(context)), new XElement("PhonemeSets", from phonemeSet in phonologicalData.PhonemeSetsOS - select ExportPhonemeSet(phonemeSet, mode)), + select ExportPhonemeSet(phonemeSet, mode, useMultiStrings)), new XElement("FeatureConstraints", from featureConstraint in phonologicalData.FeatConstraintsOS select ExportFeatureConstraint(featureConstraint)), new XElement("PhonRules", from phonRule in phonologicalData.PhonRulesOS - where !phonRule.Disabled - select ExportPhonRule(phonRule, mode)), - ExportPhonRuleFeats(phonologicalData, mode), + select ExportPhonRule(phonRule, mode, useMultiStrings, phonology)), + // Don't export PhonRuleFeats if we are only doing phonology + // because they can contain references to non-phonological data. + phonology ? null : ExportPhonRuleFeats(phonologicalData, mode, useMultiStrings), new XElement("PhIters"), new XElement("PhIters"), new XElement("PhIters"), @@ -487,25 +510,53 @@ select ExportPhonRule(phonRule, mode)), new XElement("PhIters")); } - private static XElement ExportPhonRuleFeats(IPhPhonData phonData, Normalizer.UNormalizationMode mode) + private static IEnumerable GetPhEnvironments(IPhPhonData phonologicalData) + { + return phonologicalData.Services.GetInstance().AllValidInstances(); + } + + private static XElement ExportPhEnvironment(IPhEnvironment goodEnvironment, Normalizer.UNormalizationMode mode, bool useMultiStrings = false) + { + if (useMultiStrings) + { + return new XElement("PhEnvironment", + new XAttribute("Id", goodEnvironment.Hvo), + CreateAttribute("LeftContext", goodEnvironment.LeftContextRA), + CreateAttribute("RightContext", goodEnvironment.RightContextRA), + ExportMultiString(goodEnvironment.Name, "Name", goodEnvironment), + ExportMultiString(goodEnvironment.Description, "Description", goodEnvironment), + ExportTsString(goodEnvironment.StringRepresentation, "StringRepresentation", goodEnvironment)); + } + return new XElement("PhEnvironment", + new XAttribute("Id", goodEnvironment.Hvo), + new XAttribute("StringRepresentation", + Normalize(goodEnvironment.StringRepresentation, mode)), + CreateAttribute("LeftContext", goodEnvironment.LeftContextRA), + CreateAttribute("RightContext", goodEnvironment.RightContextRA), + ExportBestAnalysis(goodEnvironment.Name, "Name", mode), + ExportBestAnalysis(goodEnvironment.Description, "Description", mode)); + } + + private static XElement ExportPhonRuleFeats(IPhPhonData phonData, Normalizer.UNormalizationMode mode, bool useMultiStrings) { return new XElement("PhonRuleFeats", from phonRuleFeat in phonData.PhonRuleFeatsOA.ReallyReallyAllPossibilities - select ExportPhonRuleFeat(phonRuleFeat as IPhPhonRuleFeat, mode)); + select ExportPhonRuleFeat(phonRuleFeat as IPhPhonRuleFeat, mode, useMultiStrings)); } - private static XElement ExportPhonRuleFeat(IPhPhonRuleFeat phonRuleFeat, Normalizer.UNormalizationMode mode) + private static XElement ExportPhonRuleFeat(IPhPhonRuleFeat phonRuleFeat, Normalizer.UNormalizationMode mode, bool useMultiStrings) { return new XElement("PhonRuleFeat", new XAttribute("Id", phonRuleFeat.Hvo), - ExportBestAnalysis(phonRuleFeat.Name, "Name", mode), + ExportBestAnalysis(phonRuleFeat.Name, "Name", mode, useMultiStrings, phonRuleFeat), new XElement("Item", phonRuleFeat.ItemRA != null ? new XAttribute("itemRef", phonRuleFeat.ItemRA.Hvo) : new XAttribute("missing", 1))); } - private static XElement ExportPhonRule(IPhSegmentRule phonRule, Normalizer.UNormalizationMode mode) + private static XElement ExportPhonRule(IPhSegmentRule phonRule, Normalizer.UNormalizationMode mode, bool useMultiStrings = false, bool phonology = false) { - if (phonRule.Disabled) + if (phonRule.Disabled && !phonology) + // Don't export disabled rules unless you are exporting the phonology. return null; XElement retVal = null; switch (phonRule.ClassName) @@ -514,34 +565,36 @@ private static XElement ExportPhonRule(IPhSegmentRule phonRule, Normalizer.UNorm var asMetathesisRule = (IPhMetathesisRule)phonRule; retVal = new XElement("PhMetathesisRule", new XAttribute("Id", phonRule.Hvo), + phonRule.Disabled ? new XAttribute("Disabled", phonRule.Disabled) : null, new XAttribute("Direction", phonRule.Direction), - ExportBestAnalysis(phonRule.Name, "Name", mode), - ExportBestAnalysis(phonRule.Description, "Description", mode), + ExportBestAnalysis(phonRule.Name, "Name", mode, useMultiStrings, phonRule), + ExportBestAnalysis(phonRule.Description, "Description", mode, useMultiStrings, phonRule), new XElement("StrucDesc", ExportContextList(phonRule.StrucDescOS)), new XElement("StrucChange", asMetathesisRule.StrucChange.Text)); break; case "PhRegularRule": - var asRegularRule = (IPhRegularRule) phonRule; + var asRegularRule = (IPhRegularRule)phonRule; var constraints = new List(asRegularRule.FeatureConstraints); retVal = new XElement("PhRegularRule", new XAttribute("Id", phonRule.Hvo), + phonRule.Disabled ? new XAttribute("Disabled", phonRule.Disabled) : null, new XAttribute("Direction", phonRule.Direction), - ExportBestAnalysis(phonRule.Name, "Name", mode), - ExportBestAnalysis(phonRule.Description, "Description", mode), + ExportBestAnalysis(phonRule.Name, "Name", mode, useMultiStrings, phonRule), + ExportBestAnalysis(phonRule.Description, "Description", mode, useMultiStrings, phonRule), new XElement("StrucDesc", ExportContextList(phonRule.StrucDescOS)), from constraint in constraints select ExportItemAsReference(constraint, constraints.IndexOf(constraint), "FeatureConstraints"), - new XElement("RightHandSides", from rhs in asRegularRule.RightHandSidesOS + new XElement("RightHandSides", from rhs in asRegularRule.RightHandSidesOS select new XElement("PhSegRuleRHS", new XAttribute("Id", rhs.Hvo), new XElement("StrucChange", from structChange in rhs.StrucChangeOS - select ExportContext(structChange)), + select ExportContext(structChange)), new XElement("InputPOSes", from pos in rhs.InputPOSesRC - select ExportItemAsReference(pos, "RequiredPOS")), + select ExportItemAsReference(pos, "RequiredPOS")), new XElement("ReqRuleFeats", from rrf in rhs.ReqRuleFeatsRC - select ExportItemAsReference(rrf, "RuleFeat")), + select ExportItemAsReference(rrf, "RuleFeat")), new XElement("ExclRuleFeats", from erf in rhs.ExclRuleFeatsRC select ExportItemAsReference(erf, "RuleFeat")), new XElement("LeftContext", ExportContext(rhs.LeftContextOA)), @@ -550,10 +603,11 @@ select ExportItemAsReference(erf, "RuleFeat")), case "PhSegmentRule": retVal = new XElement("PhSegmentRule", new XAttribute("Id", phonRule.Hvo), + phonRule.Disabled ? new XAttribute("Disabled", phonRule.Disabled) : null, new XAttribute("Direction", phonRule.Direction), CreateAttribute("ord", phonRule.IndexInOwner), - ExportBestAnalysis(phonRule.Name, "Name", mode), - ExportBestAnalysis(phonRule.Description, "Description", mode), + ExportBestAnalysis(phonRule.Name, "Name", mode, useMultiStrings, phonRule), + ExportBestAnalysis(phonRule.Description, "Description", mode, useMultiStrings, phonRule), new XElement("StrucDesc", ExportContextList(phonRule.StrucDescOS))); break; @@ -574,46 +628,48 @@ private static XElement ExportFeatureConstraint(IPhFeatureConstraint featureCons ExportItemAsReference(featureConstraint.FeatureRA, "Feature")); } - private static XElement ExportPhonemeSet(IPhPhonemeSet phonemeSet, Normalizer.UNormalizationMode mode) + private static XElement ExportPhonemeSet(IPhPhonemeSet phonemeSet, Normalizer.UNormalizationMode mode, bool useMultiStrings = false) { return new XElement("PhPhonemeSet", new XAttribute("Id", phonemeSet.Hvo), - ExportBestAnalysis(phonemeSet.Name, "Name", mode), - ExportBestAnalysis(phonemeSet.Description, "Description", mode), + ExportBestAnalysis(phonemeSet.Name, "Name", mode, useMultiStrings, phonemeSet), + ExportBestAnalysis(phonemeSet.Description, "Description", mode, useMultiStrings, phonemeSet), new XElement("Phonemes", from phoneme in phonemeSet.PhonemesOC - select ExportPhoneme(phoneme, mode)), + select ExportPhoneme(phoneme, mode, useMultiStrings)), new XElement("BoundaryMarkers", from marker in phonemeSet.BoundaryMarkersOC - select ExportBoundaryMarker(marker, mode))); + select ExportBoundaryMarker(marker, mode, useMultiStrings))); } - private static XElement ExportBoundaryMarker(IPhBdryMarker bdryMarker, Normalizer.UNormalizationMode mode) + private static XElement ExportBoundaryMarker(IPhBdryMarker bdryMarker, Normalizer.UNormalizationMode mode, bool useMultiStrings = false) { return new XElement("PhBdryMarker", new XAttribute("Id", bdryMarker.Hvo), new XAttribute("Guid", bdryMarker.Guid.ToString()), - ExportBestAnalysis(bdryMarker.Name, "Name", mode), - ExportCodes(bdryMarker.CodesOS, mode)); + ExportBestAnalysis(bdryMarker.Name, "Name", mode, useMultiStrings, bdryMarker), + ExportCodes(bdryMarker.CodesOS, mode, useMultiStrings)); } - private static XElement ExportPhoneme(IPhPhoneme phoneme, Normalizer.UNormalizationMode mode) + private static XElement ExportPhoneme(IPhPhoneme phoneme, Normalizer.UNormalizationMode mode, bool useMultiStrings = false) { return new XElement("PhPhoneme", new XAttribute("Id", phoneme.Hvo), - ExportBestVernacular(phoneme.Name, "Name", mode), - ExportBestAnalysis(phoneme.Description, "Description", mode), - ExportCodes(phoneme.CodesOS, mode), - new XElement("BasicIPASymbol", phoneme.BasicIPASymbol.Text), + ExportBestVernacular(phoneme.Name, "Name", mode, useMultiStrings, phoneme), + ExportBestAnalysis(phoneme.Description, "Description", mode, useMultiStrings, phoneme), + ExportCodes(phoneme.CodesOS, mode, useMultiStrings), + useMultiStrings ? + ExportTsString(phoneme.BasicIPASymbol, "BasicIPASymbol", phoneme) : + new XElement("BasicIPASymbol", phoneme.BasicIPASymbol.Text), new XElement("PhonologicalFeatures", ExportFeatureStructure(phoneme.FeaturesOA))); } - private static XElement ExportCodes(IEnumerable codes, Normalizer.UNormalizationMode mode) + private static XElement ExportCodes(IEnumerable codes, Normalizer.UNormalizationMode mode, bool useMultiStrings = false) { return new XElement("Codes", from phone in codes.Where(phone => !string.IsNullOrEmpty(phone.Representation.BestVernacularAnalysisAlternative.Text)) select new XElement("PhCode", new XAttribute("Id", phone.Hvo), ExportBestVernacularOrAnalysis(phone.Representation, - "Representation", mode))); + "Representation", mode, useMultiStrings, phone))); } private static XElement ExportContext(IPhContextOrVar context) @@ -671,13 +727,13 @@ from minus in asPhSimpleContextNC.MinusConstrRS return retVal; } - private static XElement ExportNaturalClass(IPhNaturalClass naturalClass, Normalizer.UNormalizationMode mode) + private static XElement ExportNaturalClass(IPhNaturalClass naturalClass, Normalizer.UNormalizationMode mode, bool useMultiStrings = false) { return new XElement(naturalClass.ClassName, new XAttribute("Id", naturalClass.Hvo), - ExportBestAnalysis(naturalClass.Name, "Name", mode), - ExportBestAnalysis(naturalClass.Description, "Description", mode), - ExportBestAnalysis(naturalClass.Abbreviation, "Abbreviation", mode), + ExportBestAnalysis(naturalClass.Name, "Name", mode, useMultiStrings, naturalClass), + ExportBestAnalysis(naturalClass.Description, "Description", mode, useMultiStrings, naturalClass), + ExportBestAnalysis(naturalClass.Abbreviation, "Abbreviation", mode, useMultiStrings, naturalClass), (naturalClass is IPhNCFeatures) ? ExportNaturalClassContents(naturalClass as IPhNCFeatures) : ExportNaturalClassContents(naturalClass as IPhNCSegments)); @@ -738,7 +794,7 @@ private static XElement ExportStemName(IMoStemName stemName, Normalizer.UNormali select ExportFeatureStructure(region))); } - private static XElement ExportFeatureSystem(IFsFeatureSystem featureSystem, string elementName, Normalizer.UNormalizationMode mode) + private static XElement ExportFeatureSystem(IFsFeatureSystem featureSystem, string elementName, Normalizer.UNormalizationMode mode, bool useMultiStrings = false) { return new XElement(elementName, new XAttribute("Id", featureSystem.Hvo), @@ -746,18 +802,18 @@ private static XElement ExportFeatureSystem(IFsFeatureSystem featureSystem, stri from type in featureSystem.TypesOC select new XElement("FsFeatStrucType", new XAttribute("Id", type.Hvo), - ExportBestAnalysis(type.Name, "Name", mode), - ExportBestAnalysis(type.Description, "Description", mode), - ExportBestAnalysis(type.Abbreviation, "Abbreviation", mode), + ExportBestAnalysis(type.Name, "Name", mode, useMultiStrings, featureSystem), + ExportBestAnalysis(type.Description, "Description", mode, useMultiStrings, featureSystem), + ExportBestAnalysis(type.Abbreviation, "Abbreviation", mode, useMultiStrings, featureSystem), new XElement("Features", from featureRef in type.FeaturesRS select ExportItemAsReference(featureRef, "Feature")))), new XElement("Features", from featDefn in featureSystem.FeaturesOC - select ExportFeatureDefn(featDefn, mode))); + select ExportFeatureDefn(featDefn, mode, useMultiStrings))); } - private static XElement ExportFeatureDefn(IFsFeatDefn featureDefn, Normalizer.UNormalizationMode mode) + private static XElement ExportFeatureDefn(IFsFeatDefn featureDefn, Normalizer.UNormalizationMode mode, bool useMultiStrings = false) { switch (featureDefn.ClassName) { @@ -768,23 +824,23 @@ private static XElement ExportFeatureDefn(IFsFeatDefn featureDefn, Normalizer.UN var closedFD = (IFsClosedFeature)featureDefn; return new XElement("FsClosedFeature", new XAttribute("Id", featureDefn.Hvo), - ExportBestAnalysis(featureDefn.Name, "Name", mode), - ExportBestAnalysis(featureDefn.Description, "Description", mode), - ExportBestAnalysis(closedFD.Abbreviation, "Abbreviation", mode), + ExportBestAnalysis(featureDefn.Name, "Name", mode, useMultiStrings, featureDefn), + ExportBestAnalysis(featureDefn.Description, "Description", mode, useMultiStrings, featureDefn), + ExportBestAnalysis(closedFD.Abbreviation, "Abbreviation", mode, useMultiStrings, featureDefn), new XElement("Values", from value in closedFD.ValuesOC select new XElement("FsSymFeatVal", new XAttribute("Id", value.Hvo), - ExportBestAnalysis(value.Name, "Name", mode), - ExportBestAnalysis(value.Description, "Description", mode), - ExportBestAnalysis(value.Abbreviation, "Abbreviation", mode)))); + ExportBestAnalysis(value.Name, "Name", mode, useMultiStrings, value), + ExportBestAnalysis(value.Description, "Description", mode, useMultiStrings, featureDefn), + ExportBestAnalysis(value.Abbreviation, "Abbreviation", mode, useMultiStrings, value)))); case "FsComplexFeature": var complexFD = (IFsComplexFeature) featureDefn; return new XElement("FsComplexFeature", new XAttribute("Id", featureDefn.Hvo), - ExportBestAnalysis(featureDefn.Name, "Name", mode), - ExportBestAnalysis(featureDefn.Description, "Description", mode), - ExportBestAnalysis(complexFD.Abbreviation, "Abbreviation", mode), + ExportBestAnalysis(featureDefn.Name, "Name", mode, useMultiStrings, featureDefn), + ExportBestAnalysis(featureDefn.Description, "Description", mode, useMultiStrings, featureDefn), + ExportBestAnalysis(complexFD.Abbreviation, "Abbreviation", mode, useMultiStrings, featureDefn), ExportItemAsReference(complexFD.TypeRA, "Type")); } } @@ -866,28 +922,66 @@ from sfxslot in template.SuffixSlotsRS where IsValidSlot(sfxslot) select ExportItemAsReference(sfxslot, template.PrefixSlotsRS.IndexOf(sfxslot), "SuffixSlots")); } - private static XElement ExportBestAnalysis(IMultiAccessorBase multiString, string elementName, Normalizer.UNormalizationMode mode) + private static XElement ExportTsString(ITsString tsString, string elementName, ICmObject obj) + { + if (tsString == null) throw new ArgumentNullException("tsString"); + if (String.IsNullOrEmpty(elementName)) throw new ArgumentNullException("elementName"); + if (obj == null) throw new ArgumentNullException("obj"); + var writingSystemManager = obj.Cache.WritingSystemFactory; + string xml = TsStringSerializer.SerializeTsStringToXml(tsString, writingSystemManager); + return new XElement(elementName, XElement.Parse(xml)); + } + + private static IEnumerable ExportMultiString(IMultiAccessorBase multiString, string elementName, ICmObject obj) { if (multiString == null) throw new ArgumentNullException("multiString"); if (String.IsNullOrEmpty(elementName)) throw new ArgumentNullException("elementName"); + if (obj == null) throw new ArgumentNullException("obj"); + if (multiString.StringCount == 0) + return null; + using (var memoryStream = new MemoryStream()) + { + using (var writer = XmlServices.CreateWriter(memoryStream)) + { + ReadWriteServices.WriteMultiFoo(writer, elementName, (MultiAccessor)multiString); + writer.Flush(); + } + var bytes = memoryStream.ToArray(); + string xml = Encoding.UTF8.GetString(bytes); + return new List { XElement.Parse(xml) }; + } + } - return new XElement(elementName, Normalize(multiString.BestAnalysisAlternative, mode)); + private static IEnumerable ExportBestAnalysis(IMultiAccessorBase multiString, string elementName, Normalizer.UNormalizationMode mode, + bool useMultiStrings = false, ICmObject obj = null) + { + if (useMultiStrings) + return ExportMultiString(multiString, elementName, obj); + if (multiString == null) throw new ArgumentNullException("multiString"); + if (String.IsNullOrEmpty(elementName)) throw new ArgumentNullException("elementName"); + return new List { new XElement(elementName, Normalize(multiString.BestAnalysisAlternative, mode)) }; } - private static XElement ExportBestVernacular(IMultiAccessorBase multiString, string elementName, Normalizer.UNormalizationMode mode) + private static IEnumerable ExportBestVernacular(IMultiAccessorBase multiString, string elementName, Normalizer.UNormalizationMode mode, + bool useMultiStrings = false, ICmObject obj = null) { + if (useMultiStrings) + return ExportMultiString(multiString, elementName, obj); if (multiString == null) throw new ArgumentNullException("multiString"); if (String.IsNullOrEmpty(elementName)) throw new ArgumentNullException("elementName"); - return new XElement(elementName, Normalize(multiString.BestVernacularAlternative, mode)); + return new List { new XElement(elementName, Normalize(multiString.BestVernacularAlternative, mode)) }; } - private static XElement ExportBestVernacularOrAnalysis(IMultiAccessorBase multiString, string elementName, Normalizer.UNormalizationMode mode) + private static IEnumerable ExportBestVernacularOrAnalysis(IMultiAccessorBase multiString, string elementName, Normalizer.UNormalizationMode mode, + bool useMultiStrings = false, ICmObject obj = null) { + if (useMultiStrings) + return ExportMultiString(multiString, elementName, obj); if (multiString == null) throw new ArgumentNullException("multiString"); if (String.IsNullOrEmpty(elementName)) throw new ArgumentNullException("elementName"); - return new XElement(elementName, Normalize(multiString.BestVernacularAnalysisAlternative, mode)); + return new List { new XElement(elementName, Normalize(multiString.BestVernacularAnalysisAlternative, mode)) }; } private static string Normalize(ITsString text, Normalizer.UNormalizationMode mode) diff --git a/src/SIL.LCModel/DomainServices/PhonologyServices.cs b/src/SIL.LCModel/DomainServices/PhonologyServices.cs new file mode 100644 index 00000000..6449cafa --- /dev/null +++ b/src/SIL.LCModel/DomainServices/PhonologyServices.cs @@ -0,0 +1,115 @@ +using SIL.LCModel.Infrastructure; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Xml; +using System.Xml.Linq; +using SIL.LCModel.Application.ApplicationServices; +using SIL.LCModel.Core.Text; +using SIL.LCModel.Infrastructure.Impl; +using static Icu.Normalization.Normalizer2; + +namespace SIL.LCModel.DomainServices +{ + public class PhonologyServices + { + public PhonologyServices(LcmCache cache, string wsVernId = null) + { + Cache = cache; + m_wsVernId = wsVernId; + } + + public static string VersionId = "1"; + + LcmCache Cache { get; set; } + + private readonly string m_wsVernId; + + /// + /// Export the phonology to a file in the Phonology format. + /// The Phonology format is similar to M3Dump except that strings are exported as multi-strings. + /// + /// + public void ExportPhonologyAsXml(string filename) + { + var xml = ExportPhonologyAsXml(); + xml.Save(filename); + } + + /// + /// Export the phonology as an XML Document. + /// + /// + public XDocument ExportPhonologyAsXml() + { + return M3ModelExportServices.ExportPhonology(Cache.LanguageProject); + } + + /// + /// Import phonology from the give XML file. + /// + /// + public void ImportPhonologyFromXml(string sFilename) + { + // Import the Phonology format using ImportData. + // ImportData has been extended to accept the Phonology format. + XmlImportData xid = new XmlImportData(Cache, true); + xid.ImportData(sFilename, null); + } + + /// + /// Import phonology from the given text reader. + /// + /// + public void ImportPhonologyFromXml(TextReader rdr) + { + // Import the Phonology format using ImportData. + // ImportData has been extended to accept the Phonology format. + XmlImportData xid = new XmlImportData(Cache, true); + xid.ImportData(rdr, null, null); + // NonUndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW(Cache.ActionHandlerAccessor, + // () => AssignVernacularWritingSystemToDefaultPhPhonemes(Cache)); + } + + private void AssignVernacularWritingSystemToDefaultPhPhonemes(LcmCache cache) + { + // For all PhCodes in the default phoneme set, change the writing system from "en" to icuLocale + if (cache.LanguageProject.PhonologicalDataOA.PhonemeSetsOS.Count == 0) + return; + var phSet = cache.LanguageProject.PhonologicalDataOA.PhonemeSetsOS[0]; + int wsVern = m_wsVernId == null + ? cache.DefaultVernWs + : cache.ServiceLocator.WritingSystemManager.Get(m_wsVernId).Handle; + foreach (var phone in phSet.PhonemesOC) + { + foreach (var code in phone.CodesOS) + { + + if (code.Representation.VernacularDefaultWritingSystem.Length == 0) + code.Representation.VernacularDefaultWritingSystem = + TsStringUtils.MakeString(code.Representation.UserDefaultWritingSystem.Text, wsVern); + } + if (phone.Name.VernacularDefaultWritingSystem.Length == 0) + phone.Name.VernacularDefaultWritingSystem = + TsStringUtils.MakeString(phone.Name.UserDefaultWritingSystem.Text, wsVern); + } + foreach (var mrkr in phSet.BoundaryMarkersOC) + { + foreach (var code in mrkr.CodesOS) + { + if (code.Representation.VernacularDefaultWritingSystem.Length == 0) + code.Representation.VernacularDefaultWritingSystem = + TsStringUtils.MakeString(code.Representation.UserDefaultWritingSystem.Text, wsVern); + } + if (mrkr.Name.VernacularDefaultWritingSystem.Length == 0) + mrkr.Name.VernacularDefaultWritingSystem = + TsStringUtils.MakeString(mrkr.Name.UserDefaultWritingSystem.Text, wsVern); + } + } + + + } +} diff --git a/tests/SIL.LCModel.Tests/DomainServices/PhonologyServicesTest.cs b/tests/SIL.LCModel.Tests/DomainServices/PhonologyServicesTest.cs new file mode 100644 index 00000000..83ac1d96 --- /dev/null +++ b/tests/SIL.LCModel.Tests/DomainServices/PhonologyServicesTest.cs @@ -0,0 +1,1193 @@ +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Linq; +using SIL.LCModel.Core.WritingSystems; +using SIL.LCModel.Infrastructure; +using System.IO; +using SIL.LCModel.Core.KernelInterfaces; +using StructureMap.Diagnostics.TreeView; +using SIL.LCModel.Core.Text; +using System.Xml.Linq; +using System.Security.Cryptography; +using SIL.Xml; +using System.Xml; +using static Icu.Normalization.Normalizer2; +using SIL.LCModel.Infrastructure.Impl; + +namespace SIL.LCModel.DomainServices +{ + [TestFixture] + public class PhonologyServicesTest + { + private LcmCache m_cache; + private DateTime m_now; + + ///-------------------------------------------------------------------------------------- + /// + /// Create the cache before each test + /// + ///-------------------------------------------------------------------------------------- + [SetUp] + public void CreateTestCache() + { + m_now = DateTime.Now; + m_cache = LcmCache.CreateCacheWithNewBlankLangProj(new TestProjectId(BackendProviderType.kMemoryOnly, "MemoryOnly.mem"), + "en", "fr", "en", new DummyLcmUI(), TestDirectoryFinder.LcmDirectories, new LcmSettings()); + IDataSetup dataSetup = m_cache.ServiceLocator.GetInstance(); + dataSetup.LoadDomain(BackendBulkLoadDomain.All); + if (m_cache.LangProject != null) + { + if (m_cache.LangProject.DefaultVernacularWritingSystem == null) + { + List rglgws = m_cache.ServiceLocator.WritingSystemManager.WritingSystems.ToList(); + if (rglgws.Count > 0) + { + m_cache.DomainDataByFlid.BeginNonUndoableTask(); + m_cache.LangProject.DefaultVernacularWritingSystem = rglgws[rglgws.Count - 1]; + m_cache.DomainDataByFlid.EndNonUndoableTask(); + } + } + if (m_cache.LangProject.DefaultAnalysisWritingSystem == null) + { + List rglgws = m_cache.ServiceLocator.WritingSystemManager.WritingSystems.ToList(); + if (rglgws.Count > 0) + { + m_cache.DomainDataByFlid.BeginNonUndoableTask(); + m_cache.LangProject.DefaultAnalysisWritingSystem = rglgws[rglgws.Count - 1]; + m_cache.DomainDataByFlid.EndNonUndoableTask(); + } + } + } + } + + ///-------------------------------------------------------------------------------------- + /// + /// Destroy the cache after each test + /// + ///-------------------------------------------------------------------------------------- + [TearDown] + public void DestroyTestCache() + { + if (m_cache != null) + { + m_cache.Dispose(); + m_cache = null; + } + } + + private void SetDefaultVernacularWritingSystem(LcmCache cache, CoreWritingSystemDefinition vernWritingSystem) + { + var vernWsName = vernWritingSystem.Id; + var wsManager = cache.ServiceLocator.WritingSystemManager; + if (wsManager.Exists(vernWsName)) + vernWritingSystem = wsManager.Get(vernWsName); + else + { + vernWritingSystem = wsManager.Set(vernWsName); + } + NonUndoableUnitOfWorkHelper.Do(m_cache.ActionHandlerAccessor, () => + m_cache.ServiceLocator.WritingSystems.DefaultVernacularWritingSystem = vernWritingSystem); + } + + private void TestProject(string projectsDirectory, string dbFileName) + { + var projectId = new TestProjectId(BackendProviderType.kXML, dbFileName); + var m_ui = new DummyLcmUI(); + var m_lcmDirectories = new TestLcmDirectories(projectsDirectory); + using (var cache = LcmCache.CreateCacheFromExistingData(projectId, "en", m_ui, m_lcmDirectories, new LcmSettings(), + new DummyProgressDlg())) + { + // Export project as XML. + var services = new PhonologyServices(cache); + XDocument xdoc = services.ExportPhonologyAsXml(); + var xml = xdoc.ToString(); + // Import XML to a new cache and export it a second time. + XDocument xdoc2 = null; + using (var rdr = new StringReader(xml)) + { + var vernWs = cache.ServiceLocator.WritingSystemManager.Get(cache.DefaultVernWs); + SetDefaultVernacularWritingSystem(m_cache, vernWs); + var services2 = new PhonologyServices(m_cache, vernWs.Id); + services2.ImportPhonologyFromXml(rdr); + xdoc2 = services2.ExportPhonologyAsXml(); + } + var xml2 = xdoc2.ToString(); + // Compare original XML to new XML. + TestXml(xdoc, xdoc2); + } + } + + private void TestXml(string xml, string vernWs) + { + var xdoc = XDocument.Parse(xml); + NonUndoableUnitOfWorkHelper.Do(m_cache.ActionHandlerAccessor, () => + { + m_cache.ServiceLocator.WritingSystems.DefaultVernacularWritingSystem = + m_cache.ServiceLocator.WritingSystemManager.Get(vernWs); + }); + var services = new PhonologyServices(m_cache); + using (var rdr = new StringReader(xml)) + { + services.ImportPhonologyFromXml(rdr); + var xdoc2 = services.ExportPhonologyAsXml(); + var xml2 = xdoc2.ToString(); + TestXml(xdoc, xdoc2); + } + } + + private void TestXml(XDocument xdoc, XDocument xdoc2) + { + IDictionary dstMap = new Dictionary(); + IDictionary idMap1 = new Dictionary(); + IDictionary idMap2 = new Dictionary(); + TestXml(xdoc.Elements(), xdoc2.Elements(), dstMap, idMap1, idMap2); + // Test dst references. + foreach (var dst1 in dstMap.Keys) + { + var dst2 = dstMap[dst1]; + TestXml(idMap1[dst1], idMap2[dst2], dstMap, idMap1, idMap2); + } + } + + private void TestXml(IEnumerable elements, IEnumerable elements2, + IDictionary dstMap, + IDictionary idMap1, + IDictionary idMap2) + { + Assert.AreEqual(elements.Count(), elements2.Count()); + foreach (var pair in elements.Zip(elements2, Tuple.Create)) + { + TestXml(pair.Item1, pair.Item2, dstMap, idMap1, idMap2); + } + } + + private void TestXml(XElement element, XElement element2, + IDictionary dstMap, + IDictionary idMap1, + IDictionary idMap2) + { + // Check attributes. + Assert.AreEqual(element.Attributes().Count(), element2.Attributes().Count()); + foreach (var attr in element.Attributes()) + { + XName name = attr.Name; + bool found = false; + if (name == "Guid") + continue; + foreach (var attr2 in element2.Attributes()) + { + if (attr2.Name == name) + { + if (name == "Id") + { + // Save for later. + if (idMap1.ContainsKey(attr.Value)) + Assert.AreEqual(idMap1[attr.Value], element); + else + idMap1[attr.Value] = element; + if (idMap2.ContainsKey(attr2.Value)) + Assert.AreEqual(idMap2[attr2.Value], element2); + else + idMap2[attr2.Value] = element2; + } + else if (name == "dst" || name == "Feature" || name == "Value") + { + // Save for later. + if (dstMap.ContainsKey(attr.Value)) + Assert.AreEqual(dstMap[attr.Value], attr2.Value); + else + dstMap[attr.Value] = attr2.Value; + } + else + { + Assert.AreEqual(attr.Value, attr2.Value, "Attribute " + name + " has different values."); + } + found = true; + } + } + Assert.IsTrue(found); + } + // Check elements. + TestXml(element.Elements(), element2.Elements(), dstMap, idMap1, idMap2); + } + + string SpanishPhonology = @" + + + + + + /n_ + + + + + + + /[C]_ + + + + + + + + Consonants + + + + Consonants + + + + C + + + + + + + + + + + + + + + + + + + + + + + + + + + + Vowels + + + + Vowels + + + + V + + + + + + + + + + + + + Main phoneme set + + + + Main phoneme set + + + + + + ñ + + + + Voiced palatal nasal + + + + + + ñ + + + + + + ɲ + + + + + + + rr + + + + Voiced alveolar trill + + + + + + rr + + + + + + r + + + + + + + ch + + + + Voiceless alveolar affricate + + + + + + ch + + + + + + tʃ + + + + + + + y + + + + Voiced palatal approximant + + + + + + y + + + + + + j + + + + + + + a + a + + + + low central unrounded vowel + + + + + + a + a + + + + + + + + + + + + + b + b + + + + voiced bilabial stop + + + + + + b + b + + + + + + + + + + + + + d + d + + + + voiced alveolar stop + + + + + + d + d + + + + + + + + + + + + + e + e + + + + mid front unrounded vowel + + + + + + e + e + + + + + + + + + + + + + f + f + + + + voiceless labiodental fricative + + + + + + f + f + + + + + + + + + + + + + g + g + + + + voiced velar stop + + + + + + g + g + + + + + + + + + + + + + i + i + + + + high front unrounded vowel + + + + + + i + i + + + + + + + + + + + + + j + j + + + + palatal approximant + + + + + + j + j + + + + + + + + + + + + + k + k + + + + voiceless velar stop + + + + + + k + k + + + + + + + + + + + + + l + l + + + + alveolar lateral + + + + + + l + l + + + + + + + + + + + + + m + m + + + + bilabial nasal + + + + + + m + m + + + + + + + + + + + + + n + n + + + + alveolar nasal + + + + + + n + n + + + + + + + + + + + + + o + o + + + + mid back rounded vowel + + + + + + o + o + + + + + + + + + + + + + p + p + + + + voiceless bilabial stop + + + + + + p + p + + + + + + + + + + + + + r + r + + + + alveolar flap + + + + + + r + r + + + + + + + + + + + + + s + s + + + + voiceless alveolar fricative + + + + + + s + s + + + + + + + + + + + + + t + t + + + + voiceless alveolar stop + + + + + + t + t + + + + + + + + + + + + + u + u + + + + high back rounded vowel + + + + + + u + u + + + + + + + + + + + + + v + v + + + + voiced labiodental fricative + + + + + + v + v + + + + + + + + + + + + + w + w + + + + labiovelar approximant + + + + + + w + w + + + + + + + + + + + + + x + x + + + + voiceless velar fricative + + + + + + x + x + + + + + + + + + + + + + z + z + + + + voiced alveolar fricative + + + + + + z + z + + + + + + + + + + + + + ŋ + ŋ + + + + velar nasal + + + + + + ŋ + ng + + + + + + ŋ + + + + + + + q + + + + Voiceless velar plosive + + + + + + q + + + + + + k + + + + + + + + + + + + + + + + + + + + + + + + + + + # + # + + + + + # + # + + + + + + + + + + + + + + + + + + + + +"; + + [Test] + public void TestSpanish() + { + TestXml(SpanishPhonology, "es"); + } + + [Test] + public void TestEmpty() + { + var services = new PhonologyServices(m_cache); + var xdoc = services.ExportPhonologyAsXml(); + var xml = xdoc.ToString(); + using (var rdr = new StringReader(xml)) + { + services.ImportPhonologyFromXml(rdr); + var xdoc2 = services.ExportPhonologyAsXml(); + var xml2 = xdoc2.ToString(); + Assert.AreEqual(xml, xml2); + } + } + + public static readonly string ksPhFS1 = + string.Format("mcfmajor class features“The features that represent the major classes of sounds.”[http://en.wikipedia.org/wiki/Distinctive_feature] Date accessed: 12-Feb-2009" + + "consconsonantal“Consonantal segments are produced with an audible constriction in the vocal tract, like plosives, affricates, fricatives, nasals, laterals and [r]. Vowels, glides and laryngeal segments are not consonantal.”[http://en.wikipedia.org/wiki/Distinctive_feature] Date accessed: 12-Feb-2009" + + "+positive" + + "-negative" + + "sonsonorant“This feature describes the type of oral constriction that can occur in the vocal tract. [+son] designates the vowels and sonorant consonants, which are produced without the imbalance of air pressure in the vocal tract that might cause turbulence. [-son] alternatively describes the obstruents, articulated with a noticeable turbulence caused by an imbalance of air pressure in the vocal tract.”[http://en.wikipedia.org/wiki/Distinctive_feature] Date accessed: 12-Feb-2009" + + "+positive" + + "-negative" + + "sylsyllabic“Syllabic segments may function as the nucleus of a syllable, while their counterparts, the [-syl] segments, may not.”[http://en.wikipedia.org/wiki/Distinctive_feature] Date accessed: 12-Feb-2009" + + "+positive" + + "-negative", + Environment.NewLine); + + /// ------------------------------------------------------------------------------------ + /// + /// Tests adding closed features to feature system and to a feature structure + /// + /// ------------------------------------------------------------------------------------ + [Test] + public void TestPhonologicalFeatures() + { + ILangProject lp = m_cache.LangProject; + var actionHandler = m_cache.ServiceLocator.GetInstance(); + actionHandler.BeginUndoTask("Undo doing stuff", "Redo doing stuff"); + + // ================================== + // set up phonological feature system + // ================================== + // Set up the xml fs description + XmlDocument doc = new XmlDocument(); + doc.LoadXml(ksPhFS1); + // get [consonantal:positive] + XmlNode itemValue = doc.SelectSingleNode("/item/item[1]/item[1]"); + + // Add the feature for first time + IFsFeatureSystem phfs = lp.PhFeatureSystemOA; + phfs.AddFeatureFromXml(itemValue); + // get [consonantal:negative] + itemValue = doc.SelectSingleNode("/item/item[1]/item[2]"); + phfs.AddFeatureFromXml(itemValue); + // add sonorant feature + itemValue = doc.SelectSingleNode("/item/item[2]/item[1]"); + phfs.AddFeatureFromXml(itemValue); + itemValue = doc.SelectSingleNode("/item/item[2]/item[2]"); + phfs.AddFeatureFromXml(itemValue); + // add syllabic feature + itemValue = doc.SelectSingleNode("/item/item[3]/item[1]"); + phfs.AddFeatureFromXml(itemValue); + itemValue = doc.SelectSingleNode("/item/item[3]/item[2]"); + phfs.AddFeatureFromXml(itemValue); + + // =============== + // set up phonemes + // =============== + var phonData = lp.PhonologicalDataOA; + + var phonemeset = m_cache.ServiceLocator.GetInstance().Create(); + phonData.PhonemeSetsOS.Add(phonemeset); + var phonemeM = m_cache.ServiceLocator.GetInstance().Create(); + phonemeset.PhonemesOC.Add(phonemeM); + phonemeM.Name.set_String(m_cache.DefaultUserWs, "m"); + phonemeM.FeaturesOA = m_cache.ServiceLocator.GetInstance().Create(); + var fsM = phonemeM.FeaturesOA; + var closedValue = m_cache.ServiceLocator.GetInstance().Create(); + fsM.FeatureSpecsOC.Add(closedValue); + var feat = phfs.FeaturesOC.First() as IFsClosedFeature; + closedValue.FeatureRA = feat; + closedValue.ValueRA = feat.ValuesOC.First(); + closedValue = m_cache.ServiceLocator.GetInstance().Create(); + fsM.FeatureSpecsOC.Add(closedValue); + feat = phfs.FeaturesOC.ElementAt(1) as IFsClosedFeature; + closedValue.FeatureRA = feat; + closedValue.ValueRA = feat.ValuesOC.First(); + var phonemeP = m_cache.ServiceLocator.GetInstance().Create(); + phonemeset.PhonemesOC.Add(phonemeP); + phonemeP.Name.set_String(m_cache.DefaultUserWs, "p"); + phonemeP.FeaturesOA = m_cache.ServiceLocator.GetInstance().Create(); + var fsP = phonemeP.FeaturesOA; + closedValue = m_cache.ServiceLocator.GetInstance().Create(); + fsP.FeatureSpecsOC.Add(closedValue); + feat = phfs.FeaturesOC.First() as IFsClosedFeature; + closedValue.FeatureRA = feat; + closedValue.ValueRA = feat.ValuesOC.First(); + closedValue = m_cache.ServiceLocator.GetInstance().Create(); + fsP.FeatureSpecsOC.Add(closedValue); + feat = phfs.FeaturesOC.ElementAt(1) as IFsClosedFeature; + closedValue.FeatureRA = feat; + closedValue.ValueRA = feat.ValuesOC.Last(); + + var phonemeB = m_cache.ServiceLocator.GetInstance().Create(); + phonemeset.PhonemesOC.Add(phonemeB); + phonemeB.Name.set_String(m_cache.DefaultUserWs, "b"); + phonemeB.FeaturesOA = m_cache.ServiceLocator.GetInstance().Create(); + var fsB = phonemeB.FeaturesOA; + closedValue = m_cache.ServiceLocator.GetInstance().Create(); + fsB.FeatureSpecsOC.Add(closedValue); + feat = phfs.FeaturesOC.First() as IFsClosedFeature; + closedValue.FeatureRA = feat; + closedValue.ValueRA = feat.ValuesOC.First(); + closedValue = m_cache.ServiceLocator.GetInstance().Create(); + fsB.FeatureSpecsOC.Add(closedValue); + feat = phfs.FeaturesOC.ElementAt(1) as IFsClosedFeature; + closedValue.FeatureRA = feat; + closedValue.ValueRA = feat.ValuesOC.Last(); + + // ==================== + // set up natural class + // ==================== + var natClass = m_cache.ServiceLocator.GetInstance().Create(); + phonData.NaturalClassesOS.Add(natClass); + natClass.SegmentsRC.Add(phonemeM); + natClass.SegmentsRC.Add(phonemeP); + natClass.SegmentsRC.Add(phonemeB); + + using (var cache = m_cache = LcmCache.CreateCacheWithNewBlankLangProj( + new TestProjectId(BackendProviderType.kMemoryOnly, "MemoryOnly.mem"), + "en", "fr", "en", new DummyLcmUI(), TestDirectoryFinder.LcmDirectories, new LcmSettings())) + { + var services = new PhonologyServices(m_cache); + XDocument xdoc = services.ExportPhonologyAsXml(); + XDocument xdoc2 = null; + var xml = xdoc.ToString(); + using (var rdr = new StringReader(xml)) + { + var services2 = new PhonologyServices(cache); + services2.ImportPhonologyFromXml(rdr); + xdoc2 = services2.ExportPhonologyAsXml(); + } + var xml2 = xdoc2.ToString(); + TestXml(xdoc, xdoc2); + } + } + } +}