diff --git a/src/SIL.Machine.Translation.Thot/IParallelTextCorpusExtensions.cs b/src/SIL.Machine.Translation.Thot/IParallelTextCorpusExtensions.cs new file mode 100644 index 000000000..0188fe902 --- /dev/null +++ b/src/SIL.Machine.Translation.Thot/IParallelTextCorpusExtensions.cs @@ -0,0 +1,130 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using SIL.Machine.Corpora; +using SIL.Machine.Utils; + +namespace SIL.Machine.Translation.Thot +{ + public static class IParallelTextCorpusExtensions + { + public static IParallelTextCorpus WordAlign( + this IParallelTextCorpus corpus, + ThotWordAlignmentModelType modelType = ThotWordAlignmentModelType.FastAlign, + SymmetrizationHeuristic symmetrizationHeuristic = SymmetrizationHeuristic.GrowDiagFinalAnd, + IProgress progress = null + ) => new TrainedWordAlignParallelTextCorpus(corpus, modelType, symmetrizationHeuristic, progress); + + public static IParallelTextCorpus WordAlign( + this IParallelTextCorpus corpus, + ThotSymmetrizedWordAlignmentModel model, + int batchSize = 1024 + ) + { + if (model.EmitTrainingAlignments) + return new TransductiveWordAlignParallelTextCorpus(corpus, model); + + return CorporaExtensions.WordAlign(corpus, model, batchSize); + } + + private class TransductiveWordAlignParallelTextCorpus : WordAlignParallelTextCorpusBase + { + private readonly IParallelTextCorpus _corpus; + private readonly ITransductiveWordAlignmentModel _model; + + public TransductiveWordAlignParallelTextCorpus( + IParallelTextCorpus corpus, + ITransductiveWordAlignmentModel model + ) + : base(corpus) + { + _corpus = corpus; + _model = model; + } + + public override IEnumerable GetRows(IEnumerable textIds) => + GetTransductiveRows(_corpus, _model, textIds); + } + + private class TrainedWordAlignParallelTextCorpus : WordAlignParallelTextCorpusBase + { + private readonly IParallelTextCorpus _corpus; + private readonly ThotWordAlignmentModelType _modelType; + private readonly SymmetrizationHeuristic _symmetrizationHeuristic; + private readonly IProgress _progress; + + public TrainedWordAlignParallelTextCorpus( + IParallelTextCorpus corpus, + ThotWordAlignmentModelType modelType, + SymmetrizationHeuristic symmetrizationHeuristic, + IProgress progress + ) + : base(corpus) + { + _corpus = corpus; + _modelType = modelType; + _symmetrizationHeuristic = symmetrizationHeuristic; + _progress = progress; + } + + public override IEnumerable GetRows(IEnumerable textIds) + { + // Training on only the requested texts keeps the training-alignment index in sync with the rows. + IParallelTextCorpus corpus = _corpus.FilterTexts(textIds); + // Training in the generator ties the model's lifetime to reading the rows, at the cost of + // training a new model on each iteration. + using (var model = ThotSymmetrizedWordAlignmentModel.Create(_modelType)) + { + model.Heuristic = _symmetrizationHeuristic; + // Retain the alignments computed during training so that the corpus can be aligned + // without a separate, potentially expensive, inference pass. + model.EmitTrainingAlignments = true; + using (ITrainer trainer = model.CreateTrainer(corpus)) + { + trainer.TrainAsync(_progress).GetAwaiter().GetResult(); + trainer.SaveAsync().GetAwaiter().GetResult(); + } + + foreach (ParallelTextRow row in GetTransductiveRows(corpus, model, textIds: null)) + yield return row; + } + } + } + + private static IEnumerable GetTransductiveRows( + IParallelTextCorpus corpus, + ITransductiveWordAlignmentModel model, + IEnumerable textIds + ) + { + // The training alignments are keyed by the order in which the sentence pairs were added during + // training, so the corpus the model was trained on must be iterated in full to keep the index in + // sync; rows outside the requested texts are skipped rather than filtered out. + var textIdList = textIds?.ToList(); + List rows = corpus.GetRows().ToList(); + for (int i = 0; i < rows.Count; i++) + { + ParallelTextRow row = rows[i]; + if (textIdList != null && !textIdList.Contains(row.TextId)) + continue; + + WordAlignmentMatrix alignment = model.GetTrainingAlignment(i); + WordAlignmentMatrix knownAlignment = row.CreateAlignmentMatrix(); + if (knownAlignment != null) + { + knownAlignment.PrioritySymmetrizeWith(alignment); + alignment = knownAlignment; + } + + IReadOnlyCollection wordPairs = alignment.ToAlignedWordPairs(); + if (model is IWordAlignmentModel wordAlignmentModel) + { + wordAlignmentModel.ComputeAlignedWordPairScores(row.SourceSegment, row.TargetSegment, wordPairs); + } + + row.AlignedWordPairs = wordPairs; + yield return row; + } + } + } +} diff --git a/src/SIL.Machine.Translation.Thot/SIL.Machine.Translation.Thot.csproj b/src/SIL.Machine.Translation.Thot/SIL.Machine.Translation.Thot.csproj index 88bb2fa0d..73a0adf16 100644 --- a/src/SIL.Machine.Translation.Thot/SIL.Machine.Translation.Thot.csproj +++ b/src/SIL.Machine.Translation.Thot/SIL.Machine.Translation.Thot.csproj @@ -12,7 +12,7 @@ - + diff --git a/src/SIL.Machine.Translation.Thot/Thot.cs b/src/SIL.Machine.Translation.Thot/Thot.cs index a012f8a5b..a7bf4de90 100644 --- a/src/SIL.Machine.Translation.Thot/Thot.cs +++ b/src/SIL.Machine.Translation.Thot/Thot.cs @@ -172,6 +172,21 @@ uint capacity [DllImport("thot", CallingConvention = CallingConvention.Cdecl)] public static extern uint swAlignModel_getMaxSentenceLength(IntPtr swAlignModelHandle); + [DllImport("thot", CallingConvention = CallingConvention.Cdecl)] + public static extern uint swAlignModel_getNumSentencePairs(IntPtr swAlignModelHandle); + + [DllImport("thot", CallingConvention = CallingConvention.Cdecl)] + public static extern double swAlignModel_getTrainingAlignment( + IntPtr swAlignModelHandle, + uint n, + IntPtr matrix, + ref uint iLen, + ref uint jLen + ); + + [DllImport("thot", CallingConvention = CallingConvention.Cdecl)] + public static extern void swAlignModel_setEmitTrainingAlignments(IntPtr swAlignModelHandle, bool value); + [DllImport("thot", CallingConvention = CallingConvention.Cdecl)] public static extern void swAlignModel_setVariationalBayes(IntPtr swAlignModelHandle, bool variationalBayes); diff --git a/src/SIL.Machine.Translation.Thot/ThotSymmetrizedWordAlignmentModel.cs b/src/SIL.Machine.Translation.Thot/ThotSymmetrizedWordAlignmentModel.cs new file mode 100644 index 000000000..dcc8aaa30 --- /dev/null +++ b/src/SIL.Machine.Translation.Thot/ThotSymmetrizedWordAlignmentModel.cs @@ -0,0 +1,62 @@ +namespace SIL.Machine.Translation.Thot +{ + public class ThotSymmetrizedWordAlignmentModel : SymmetrizedWordAlignmentModel, ITransductiveWordAlignmentModel + { + private readonly ThotWordAlignmentModel _directWordAlignmentModel; + private readonly ThotWordAlignmentModel _inverseWordAlignmentModel; + + public ThotSymmetrizedWordAlignmentModel( + ThotWordAlignmentModel directWordAlignmentModel, + ThotWordAlignmentModel inverseWordAlignmentModel + ) + : base(directWordAlignmentModel, inverseWordAlignmentModel) + { + _directWordAlignmentModel = directWordAlignmentModel; + _inverseWordAlignmentModel = inverseWordAlignmentModel; + } + + public bool EmitTrainingAlignments + { + get => _directWordAlignmentModel.EmitTrainingAlignments; + set + { + _directWordAlignmentModel.EmitTrainingAlignments = value; + _inverseWordAlignmentModel.EmitTrainingAlignments = value; + } + } + + public int TrainingAlignmentCount => _directWordAlignmentModel.TrainingAlignmentCount; + + public static ThotSymmetrizedWordAlignmentModel Create(ThotWordAlignmentModelType modelType) => + new ThotSymmetrizedWordAlignmentModel( + ThotWordAlignmentModel.Create(modelType), + ThotWordAlignmentModel.Create(modelType) + ); + + public WordAlignmentMatrix GetTrainingAlignment(int n) + { + WordAlignmentMatrix bestMatrix = _directWordAlignmentModel.GetTrainingAlignment(n); + if (Heuristic == SymmetrizationHeuristic.None) + return bestMatrix; + + WordAlignmentMatrix invMatrix = _inverseWordAlignmentModel.GetTrainingAlignment(n); + invMatrix.Transpose(); + + // Skip the combine when the matrices are degenerate or their dimensions don't + // line up (e.g. an out-of-range n, or a pair filtered out of training in only + // one direction): the heuristic operations require matching dimensions. + if ( + bestMatrix.RowCount == 0 + || bestMatrix.ColumnCount == 0 + || invMatrix.RowCount != bestMatrix.RowCount + || invMatrix.ColumnCount != bestMatrix.ColumnCount + ) + { + return bestMatrix; + } + + bestMatrix.SymmetrizeWith(invMatrix, Heuristic); + return bestMatrix; + } + } +} diff --git a/src/SIL.Machine.Translation.Thot/ThotWordAlignmentModel.cs b/src/SIL.Machine.Translation.Thot/ThotWordAlignmentModel.cs index a4b335eb6..14091eb1e 100644 --- a/src/SIL.Machine.Translation.Thot/ThotWordAlignmentModel.cs +++ b/src/SIL.Machine.Translation.Thot/ThotWordAlignmentModel.cs @@ -13,7 +13,10 @@ namespace SIL.Machine.Translation.Thot { - public abstract class ThotWordAlignmentModel : DisposableBase, IIbm1WordAlignmentModel + public abstract class ThotWordAlignmentModel + : DisposableBase, + ITransductiveWordAlignmentModel, + IIbm1WordAlignmentModel { public static ThotWordAlignmentModel Create(ThotWordAlignmentModelType type) { @@ -156,6 +159,30 @@ public void Save() Thot.swAlignModel_save(Handle, _prefFileName); } + public bool EmitTrainingAlignments { get; set; } + + public int TrainingAlignmentCount => (int)Thot.swAlignModel_getNumSentencePairs(Handle); + + public WordAlignmentMatrix GetTrainingAlignment(int n) + { + CheckDisposed(); + + uint iLen = 0; + uint jLen = 0; + Thot.swAlignModel_getTrainingAlignment(Handle, (uint)n, IntPtr.Zero, ref iLen, ref jLen); + + IntPtr nativeMatrix = Thot.AllocNativeMatrix((int)iLen, (int)jLen); + try + { + Thot.swAlignModel_getTrainingAlignment(Handle, (uint)n, nativeMatrix, ref iLen, ref jLen); + return Thot.ConvertNativeMatrixToWordAlignmentMatrix(nativeMatrix, iLen, jLen); + } + finally + { + Thot.FreeNativeMatrix(nativeMatrix, iLen); + } + } + public double GetTranslationScore(string sourceWord, string targetWord) { return GetTranslationProbability(sourceWord, targetWord); @@ -316,7 +343,7 @@ private class Trainer : ThotWordAlignmentModelTrainer private readonly ThotWordAlignmentModel _model; public Trainer(ThotWordAlignmentModel model, IParallelTextCorpus corpus) - : base(model.Type, corpus, model._prefFileName, model.Parameters) + : base(model.Type, corpus, model._prefFileName, model.Parameters, model.EmitTrainingAlignments) { _model = model; CloseOnDispose = false; diff --git a/src/SIL.Machine.Translation.Thot/ThotWordAlignmentModelTrainer.cs b/src/SIL.Machine.Translation.Thot/ThotWordAlignmentModelTrainer.cs index c1624d4c9..4156d9d91 100644 --- a/src/SIL.Machine.Translation.Thot/ThotWordAlignmentModelTrainer.cs +++ b/src/SIL.Machine.Translation.Thot/ThotWordAlignmentModelTrainer.cs @@ -26,9 +26,10 @@ public ThotWordAlignmentModelTrainer( string sourceFileName, string targetFileName, string prefFileName, - ThotWordAlignmentParameters parameters = null + ThotWordAlignmentParameters parameters = null, + bool emitTrainingAlignments = false ) - : this(modelType, null, prefFileName, parameters) + : this(modelType, null, prefFileName, parameters, emitTrainingAlignments) { _sourceFileName = sourceFileName; _targetFileName = targetFileName; @@ -38,7 +39,8 @@ public ThotWordAlignmentModelTrainer( ThotWordAlignmentModelType modelType, IParallelTextCorpus corpus, string prefFileName, - ThotWordAlignmentParameters parameters = null + ThotWordAlignmentParameters parameters = null, + bool emitTrainingAlignments = false ) { _prefFileName = prefFileName; @@ -47,6 +49,8 @@ public ThotWordAlignmentModelTrainer( if (parameters == null) parameters = new ThotWordAlignmentParameters(); + EmitTrainingAlignments = emitTrainingAlignments; + _models = new List<(IntPtr, int)>(); if (modelType == ThotWordAlignmentModelType.FastAlign) { @@ -197,6 +201,8 @@ public ThotWordAlignmentModelTrainer( public TrainStats Stats { get; } = new TrainStats(); + public bool EmitTrainingAlignments { get; } + public int MaxCorpusCount { get; set; } = int.MaxValue; public Task TrainAsync(IProgress progress = null, CancellationToken cancellationToken = default) @@ -243,6 +249,14 @@ void Report() => Report(); cancellationToken.ThrowIfCancellationRequested(); + if (EmitTrainingAlignments) + { + // Retain the alignments computed during training so that they can be returned without a + // separate inference pass. Only the final (most refined) model's alignments are needed, + // since that is the model used for inference. + Thot.swAlignModel_setEmitTrainingAlignments(Handle, true); + } + int trainedSegmentCount = 0; foreach ((IntPtr handle, int storedIterationCount) in _models) { diff --git a/src/SIL.Machine/Corpora/CorporaExtensions.cs b/src/SIL.Machine/Corpora/CorporaExtensions.cs index 6b0000893..5985f3c42 100644 --- a/src/SIL.Machine/Corpora/CorporaExtensions.cs +++ b/src/SIL.Machine/Corpora/CorporaExtensions.cs @@ -1282,22 +1282,20 @@ public override IEnumerable GetRows(IEnumerable textIds } } - private class WordAlignParallelTextCorpus : ParallelTextCorpusBase + private class WordAlignParallelTextCorpus : WordAlignParallelTextCorpusBase { private readonly IParallelTextCorpus _corpus; private readonly IWordAligner _aligner; private readonly int _batchSize; public WordAlignParallelTextCorpus(IParallelTextCorpus corpus, IWordAligner aligner, int batchSize) + : base(corpus) { _corpus = corpus; _aligner = aligner; _batchSize = batchSize; } - public override bool IsSourceTokenized => _corpus.IsSourceTokenized; - public override bool IsTargetTokenized => _corpus.IsTargetTokenized; - public override IEnumerable GetRows(IEnumerable textIds) { foreach (IReadOnlyList batch in _corpus.GetRows(textIds).Batch(_batchSize)) diff --git a/src/SIL.Machine/Corpora/WordAlignParallelTextCorpusBase.cs b/src/SIL.Machine/Corpora/WordAlignParallelTextCorpusBase.cs new file mode 100644 index 000000000..2b317aec9 --- /dev/null +++ b/src/SIL.Machine/Corpora/WordAlignParallelTextCorpusBase.cs @@ -0,0 +1,22 @@ +using System.Collections.Generic; + +namespace SIL.Machine.Corpora +{ + public abstract class WordAlignParallelTextCorpusBase : ParallelTextCorpusBase + { + private readonly IParallelTextCorpus _corpus; + + protected WordAlignParallelTextCorpusBase(IParallelTextCorpus corpus) + { + _corpus = corpus; + } + + public override bool IsSourceTokenized => _corpus.IsSourceTokenized; + + public override bool IsTargetTokenized => _corpus.IsTargetTokenized; + + public override int Count(bool includeEmpty = true, IEnumerable textIds = null) => + // Aligning does not add or remove rows, so counting need not align, which may train a model. + _corpus.Count(includeEmpty, textIds); + } +} diff --git a/src/SIL.Machine/Translation/ITransductiveWordAlignmentModel.cs b/src/SIL.Machine/Translation/ITransductiveWordAlignmentModel.cs new file mode 100644 index 000000000..a7aaf8cbb --- /dev/null +++ b/src/SIL.Machine/Translation/ITransductiveWordAlignmentModel.cs @@ -0,0 +1,8 @@ +namespace SIL.Machine.Translation +{ + public interface ITransductiveWordAlignmentModel + { + int TrainingAlignmentCount { get; } + WordAlignmentMatrix GetTrainingAlignment(int n); + } +} diff --git a/tests/SIL.Machine.Translation.Thot.Tests/TestHelpers.cs b/tests/SIL.Machine.Translation.Thot.Tests/TestHelpers.cs index 34d1b8815..103822f9a 100644 --- a/tests/SIL.Machine.Translation.Thot.Tests/TestHelpers.cs +++ b/tests/SIL.Machine.Translation.Thot.Tests/TestHelpers.cs @@ -12,9 +12,19 @@ public static class TestHelpers Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "data", "toy_corpus_fa"); public static string ToyCorpusFastAlignConfigFileName => Path.Combine(ToyCorpusFastAlignFolderName, "smt.cfg"); - public static IEnumerable Split(this string segment) + public static IReadOnlyList AlignmentStrings( + IParallelTextCorpus corpus, + IEnumerable? textIds = null + ) { - return segment.Split(' '); + return + [ + .. corpus + .GetRows(textIds) + .SelectMany(row => + row.AlignedWordPairs.Select(wp => new AlignedWordPair(wp.SourceIndex, wp.TargetIndex).ToString()) + ), + ]; } public static ParallelTextCorpus CreateTestParallelCorpus() @@ -22,8 +32,7 @@ public static ParallelTextCorpus CreateTestParallelCorpus() var srcCorpus = new DictionaryTextCorpus( new MemoryText( "text1", - new[] - { + [ Row(1, "isthay isyay ayay esttay-N ."), Row(2, "ouyay ouldshay esttay-V oftenyay ."), Row(3, "isyay isthay orkingway ?"), @@ -32,15 +41,14 @@ public static ParallelTextCorpus CreateTestParallelCorpus() Row(6, "orkway-N ancay ebay ardhay !"), Row(7, "ayay esttay-N ancay ebay ardhay ."), Row(8, "isthay isyay ayay ordway !"), - } + ] ) ); var trgCorpus = new DictionaryTextCorpus( new MemoryText( "text1", - new[] - { + [ Row(1, "this is a test N ."), Row(2, "you should test V often ."), Row(3, "is this working ?"), @@ -49,13 +57,66 @@ public static ParallelTextCorpus CreateTestParallelCorpus() Row(6, "work N can be hard !"), Row(7, "a test N can be hard ."), Row(8, "this is a word !"), - } + ] ) ); return new ParallelTextCorpus(srcCorpus, trgCorpus); } + public static ThotSymmetrizedWordAlignmentModel CreateTrainedModel( + IParallelTextCorpus corpus, + ThotWordAlignmentModelType modelType = ThotWordAlignmentModelType.FastAlign + ) + { + var model = ThotSymmetrizedWordAlignmentModel.Create(modelType); + model.Heuristic = SymmetrizationHeuristic.GrowDiagFinalAnd; + model.EmitTrainingAlignments = true; + using ITrainer trainer = model.CreateTrainer(corpus); + trainer.TrainAsync().GetAwaiter().GetResult(); + trainer.SaveAsync().GetAwaiter().GetResult(); + return model; + } + + public static ParallelTextCorpus CreateTwoTextParallelCorpus() + { + var src = new DictionaryTextCorpus( + new MemoryText( + "text1", + [ + new TextRow("text1", 1) { Segment = "el gato".Split(' ') }, + new TextRow("text1", 2) { Segment = "la casa".Split(' ') }, + ] + ), + new MemoryText( + "text2", + [ + new TextRow("text2", 1) { Segment = "el perro corre".Split(' ') }, + new TextRow("text2", 2) { Segment = "la mesa".Split(' ') }, + ] + ) + ); + + var trg = new DictionaryTextCorpus( + new MemoryText( + "text1", + [ + new TextRow("text1", 1) { Segment = "the cat".Split(' ') }, + new TextRow("text1", 2) { Segment = "the house".Split(' ') }, + ] + ), + new MemoryText( + "text2", + [ + new TextRow("text2", 1) { Segment = "the dog runs".Split(' ') }, + new TextRow("text2", 2) { Segment = "the table".Split(' ') }, + ] + ) + ); + + return new ParallelTextCorpus(src, trg); + } + private static TextRow Row(int rowRef, string segment) { return new TextRow("text1", rowRef) { Segment = segment.Split() }; diff --git a/tests/SIL.Machine.Translation.Thot.Tests/ThotCorpusTests.cs b/tests/SIL.Machine.Translation.Thot.Tests/ThotCorpusTests.cs new file mode 100644 index 000000000..2de17b77a --- /dev/null +++ b/tests/SIL.Machine.Translation.Thot.Tests/ThotCorpusTests.cs @@ -0,0 +1,52 @@ +using NUnit.Framework; +using SIL.Machine.Corpora; + +namespace SIL.Machine.Translation.Thot; + +[TestFixture] +public class ThotCorpusTests +{ + [TestCase(ThotWordAlignmentModelType.FastAlign)] + [TestCase(ThotWordAlignmentModelType.Ibm1)] + public void WordAlignCorpus_TransductiveMatchesInference(ThotWordAlignmentModelType modelType) + { + IParallelTextCorpus corpus = TestHelpers.CreateTestParallelCorpus(); + corpus = corpus.WordAlign(modelType); + + // For deterministic models, the alignments retained during training match those produced by a + // separate inference pass, so the transductive output must equal aligning each row directly. + IReadOnlyList transductive = TestHelpers.AlignmentStrings(corpus); + + using ThotSymmetrizedWordAlignmentModel model = TestHelpers.CreateTrainedModel( + TestHelpers.CreateTestParallelCorpus(), + modelType + ); + IReadOnlyList inference = + [ + .. TestHelpers + .CreateTestParallelCorpus() + .GetRows() + .SelectMany(row => + model + .Align(row.SourceSegment, row.TargetSegment) + .ToAlignedWordPairs() + .Select(wp => new AlignedWordPair(wp.SourceIndex, wp.TargetIndex).ToString()) + ), + ]; + Assert.That(transductive, Is.EquivalentTo(inference)); + } + + [TestCase(ThotWordAlignmentModelType.Eflomal)] + [TestCase(ThotWordAlignmentModelType.FastAlign)] + public void WordAlignCorpus_DefaultIsTransductive(ThotWordAlignmentModelType modelType) + { + IParallelTextCorpus corpus = TestHelpers.CreateTestParallelCorpus(); + corpus = corpus.WordAlign(modelType); + List rows = [.. corpus.GetRows()]; + using (Assert.EnterMultipleScope()) + { + Assert.That(rows, Has.Count.EqualTo(8)); + Assert.That(rows.Any(row => row.AlignedWordPairs.Count > 0), Is.True); + } + } +} diff --git a/tests/SIL.Machine.Translation.Thot.Tests/ThotFastAlignWordAlignmentModelTests.cs b/tests/SIL.Machine.Translation.Thot.Tests/ThotFastAlignWordAlignmentModelTests.cs index e5ae41694..859cebd69 100644 --- a/tests/SIL.Machine.Translation.Thot.Tests/ThotFastAlignWordAlignmentModelTests.cs +++ b/tests/SIL.Machine.Translation.Thot.Tests/ThotFastAlignWordAlignmentModelTests.cs @@ -1,4 +1,5 @@ using NUnit.Framework; +using SIL.Machine.Corpora; using SIL.Machine.Utils; namespace SIL.Machine.Translation.Thot; @@ -176,4 +177,89 @@ public void Constructor_ModelCorrupted() File.WriteAllText(modelPrefix + ".src", "corrupted"); Assert.Throws(() => new ThotFastAlignWordAlignmentModel(modelPrefix)); } + + [Test] + public async Task EmitTrainingAlignments_SingleDirection() + { + ParallelTextCorpus corpus = TestHelpers.CreateTestParallelCorpus(); + ParallelTextRow row = corpus.GetRows().First(); + using var model = new ThotFastAlignWordAlignmentModel(); + model.EmitTrainingAlignments = true; + ITrainer trainer = model.CreateTrainer(corpus); + await trainer.TrainAsync(); + await trainer.SaveAsync(); + using (Assert.EnterMultipleScope()) + { + Assert.That(model.TrainingAlignmentCount, Is.EqualTo(8)); + // For a deterministic model, the retained training alignment matches the inference alignment, + // and it survives the trainer being closed. + Assert.That( + model.GetTrainingAlignment(0).ValueEquals(model.Align(row.SourceSegment, row.TargetSegment)), + Is.True + ); + } + } + + [Test] + public async Task EmitTrainingAlignments_Symmetrized() + { + ParallelTextCorpus corpus = TestHelpers.CreateTestParallelCorpus(); + ParallelTextRow row = corpus.GetRows().First(); + using var model = new ThotSymmetrizedWordAlignmentModel( + new ThotFastAlignWordAlignmentModel(), + new ThotFastAlignWordAlignmentModel() + ); + model.EmitTrainingAlignments = true; + ITrainer trainer = model.CreateTrainer(corpus); + await trainer.TrainAsync(); + await trainer.SaveAsync(); + using (Assert.EnterMultipleScope()) + { + Assert.That(model.TrainingAlignmentCount, Is.EqualTo(8)); + // The C++ symmetrized transductive alignment matches the C++ symmetrized inference alignment. + Assert.That( + model.GetTrainingAlignment(0).ValueEquals(model.Align(row.SourceSegment, row.TargetSegment)), + Is.True + ); + } + } + + [Test] + public async Task EmitTrainingAlignments_Disabled() + { + ParallelTextCorpus corpus = TestHelpers.CreateTestParallelCorpus(); + using var model = new ThotFastAlignWordAlignmentModel(); + ITrainer trainer = model.CreateTrainer(corpus); + await trainer.TrainAsync(); + await trainer.SaveAsync(); + // When emission is not enabled, retrieval returns a degenerate result rather than raising. + WordAlignmentMatrix alignment = model.GetTrainingAlignment(0); + using (Assert.EnterMultipleScope()) + { + Assert.That(alignment.ColumnCount, Is.Zero); + Assert.That(alignment.RowCount, Is.Zero); + } + } + + [Test] + public void WordAlignCorpus_TransductiveTextIdsKeepIndexInSync() + { + // Filtering by text must not desync the training-alignment index: the rows for a requested text + // must get exactly the alignments they got in the unfiltered pass, not those of earlier rows. + // The model is trained up front so that both passes read the same training alignments. + IParallelTextCorpus parallelCorpus = TestHelpers.CreateTwoTextParallelCorpus(); + using ThotSymmetrizedWordAlignmentModel model = TestHelpers.CreateTrainedModel(parallelCorpus); + IParallelTextCorpus corpus = parallelCorpus.WordAlign(model); + List full = [.. corpus.GetRows()]; + IReadOnlyList text2Expected = + [ + .. full.Skip(2) + .SelectMany(row => + row.AlignedWordPairs.Select(wp => new AlignedWordPair(wp.SourceIndex, wp.TargetIndex).ToString()) + ), + ]; + + IReadOnlyList text2Actual = TestHelpers.AlignmentStrings(corpus, ["text2"]); + Assert.That(text2Actual, Is.EqualTo(text2Expected)); + } }