From 9aa1317e8b7c654115dc6404ca2fe72030ca63f5 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Wed, 12 Aug 2026 12:08:40 -0400 Subject: [PATCH 1/3] Add ability to convert usfm when updating to the versification of the update rows --- .../Corpora/ParatextProjectTextUpdaterBase.cs | 7 +- .../Corpora/UpdateUsfmParserHandler.cs | 109 +++- src/SIL.Machine/Corpora/UsfmToken.cs | 9 + .../Corpora/UpdateUsfmParserHandlerTests.cs | 468 +++++++++++++----- 4 files changed, 466 insertions(+), 127 deletions(-) diff --git a/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs b/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs index e38e528f6..fd104e3f8 100644 --- a/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs +++ b/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs @@ -33,7 +33,8 @@ public string UpdateUsfm( IEnumerable updateBlockHandlers = null, IEnumerable<(int, string)> remarks = null, Func errorHandler = null, - bool compareSegments = false + bool compareSegments = false, + bool convertUsfmToUpdateRowVersification = false ) { string fileName = _settings.GetBookFileName(bookId); @@ -57,7 +58,9 @@ public string UpdateUsfm( updateBlockHandlers, remarks, errorHandler, - compareSegments + compareSegments, + _settings.Versification, + convertUsfmToUpdateRowVersification ); try { diff --git a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs index f21be754d..6672eede6 100644 --- a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs +++ b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs @@ -49,6 +49,7 @@ public class UpdateUsfmParserHandler : ScriptureRefUsfmParserHandlerBase private int _verseRowIndex; private readonly Dictionary> _verseRowsMap; private readonly ScrVers _updateRowsVersification; + private readonly ScrVers _usfmVersification; private readonly List _tokens; private readonly List _updatedText; private readonly List _embedTokens; @@ -65,6 +66,10 @@ public class UpdateUsfmParserHandler : ScriptureRefUsfmParserHandlerBase private int _tokenIndex; private readonly Func _errorHandler; private readonly bool _compareSegments; + private readonly bool _convertUsfmToUpdateRowVersification; + private UsfmToken _currentChapterToken; + private int _currentChapterNum; + private bool _skipNextVerseText; /// UpdateUsfmRows must be in order public UpdateUsfmParserHandler( @@ -78,7 +83,9 @@ public UpdateUsfmParserHandler( IEnumerable updateBlockHandlers = null, IEnumerable<(int, string)> remarks = null, Func errorHandler = null, - bool compareSegments = false + bool compareSegments = false, + ScrVers usfmVersification = null, + bool convertUsfmToUpdateRowVersification = false ) { _rows = rows ?? Array.Empty(); @@ -89,6 +96,7 @@ public UpdateUsfmParserHandler( _updateRowsVersification = ScrVers.English; if (_rows.Count > 0) _updateRowsVersification = _rows.First(r => r.Refs.Count > 0).Refs[0].Versification; + _usfmVersification = usfmVersification ?? _updateRowsVersification; _tokens = new List(); _updatedText = new List(); _updateBlocks = new Stack(); @@ -112,6 +120,11 @@ public UpdateUsfmParserHandler( if (_errorHandler == null) _errorHandler = (error) => false; _compareSegments = compareSegments; + _convertUsfmToUpdateRowVersification = + convertUsfmToUpdateRowVersification && _updateRowsVersification != _usfmVersification; + _currentChapterToken = null; + _currentChapterNum = 0; + _skipNextVerseText = false; } public IReadOnlyList Tokens => _tokens; @@ -119,6 +132,11 @@ public UpdateUsfmParserHandler( public override void EndUsfm(UsfmParserState state) { CollectUpdatableTokens(state); + if (_currentChapterToken != null) + { + _tokens.Add(_currentChapterToken); + _currentChapterToken = null; + } base.EndUsfm(state); } @@ -148,6 +166,12 @@ public override void EndBook(UsfmParserState state, string marker) UsfmUpdateBlock updateBlock = _updateBlocks.Pop(); _tokens.AddRange(updateBlock.GetTokens()); + if (_currentChapterToken != null) + { + _tokens.Add(_currentChapterToken); + _currentChapterToken = null; + } + base.EndBook(state, marker); } @@ -398,7 +422,11 @@ protected override void StartVerseText(UsfmParserState state, IReadOnlyList scriptureRefs) { - EndUpdateBlock(state, scriptureRefs); + if (!_skipNextVerseText) + { + EndUpdateBlock(state, scriptureRefs); + _skipNextVerseText = false; + } } protected override void StartNonVerseText(UsfmParserState state, ScriptureRef scriptureRef) @@ -564,9 +592,16 @@ private void CollectUpdatableTokens(UsfmParserState state) UsfmToken token = state.Tokens[_tokenIndex]; if (token.Type == UsfmTokenType.Verse) { - string sanitizedVerseData = SanitizeVerseData(token.Data); - token = new UsfmToken(token.Type, token.Marker, token.Text, token.EndMarker, sanitizedVerseData); + VerseRef updatedVerse = UpdateVerseData(state, token); + if (updatedVerse.BookNum != state.VerseRef.BookNum) + { + _tokenIndex++; + _skipNextVerseText = true; + continue; + } + token = new UsfmToken(token.Type, token.Marker, token.Text, token.EndMarker, updatedVerse.Verse); } + if (CurrentTextType == ScriptureTextType.Embed) { _embedTokens.Add(token); @@ -582,10 +617,48 @@ private void CollectUpdatableTokens(UsfmParserState state) { _tokens.Add(token); } + _tokenIndex++; } } + private VerseRef UpdateVerseData(UsfmParserState state, UsfmToken token) + { + string updatedVerseData = SanitizeVerseData(token.Data); + VerseRef verseRef = state.VerseRef; + verseRef.Verse = updatedVerseData; + if (_convertUsfmToUpdateRowVersification) + { + verseRef = verseRef.ChangeVersificationWithSegments(_updateRowsVersification); + if (verseRef.ChapterNum != _currentChapterNum && state.VerseRef.BookNum == verseRef.BookNum) + { + if (_currentChapterToken != null) + { + _tokens.Add(_currentChapterToken.Copy()); + _currentChapterToken = null; + } + else + { + UsfmToken newChapterToken = new UsfmToken( + UsfmTokenType.Chapter, + "c", + "", + "", + verseRef.ChapterNum.ToString() + ); + _tokens.Add(newChapterToken); + } + _currentChapterNum = verseRef.ChapterNum; + } + if (verseRef.ChapterNum == verseRef.Versification.GetLastChapter(verseRef.BookNum)) + { + _currentChapterToken = null; + } + } + + return verseRef; + } + private void CollectReadonlyTokens(UsfmParserState state) { while (_tokenIndex <= state.Index + state.SpecialTokenCount) @@ -597,7 +670,16 @@ private void CollectReadonlyTokens(UsfmParserState state) } else { - _tokens.Add(token); + if (_convertUsfmToUpdateRowVersification && token.Type == UsfmTokenType.Chapter) + { + if (_currentChapterToken != null) + _tokens.Add(_currentChapterToken); + _currentChapterToken = token; + } + else + { + _tokens.Add(token); + } } _tokenIndex++; } @@ -663,10 +745,13 @@ private bool HasNewText() private void StartUpdateBlock(IReadOnlyList scriptureRefs) { (IReadOnlyList rowTexts, Dictionary metadata) = AdvanceRows(scriptureRefs); - _updateBlocks.Push( - new UsfmUpdateBlock(scriptureRefs, metadata: metadata ?? new Dictionary()) - ); - PushUpdatedText(rowTexts.Select(t => new UsfmToken(t + " "))); + if (!_skipNextVerseText) + { + _updateBlocks.Push( + new UsfmUpdateBlock(scriptureRefs, metadata: metadata ?? new Dictionary()) + ); + PushUpdatedText(rowTexts.Select(t => new UsfmToken(t + " "))); + } } private void EndUpdateBlock(UsfmParserState state, IReadOnlyList scriptureRefs) @@ -757,7 +842,11 @@ private bool IsNonverseParagraph(UsfmParserState state, UsfmUpdateBlockElement e private void UpdateVerseRowsMap() { _verseRowsMap.Clear(); - while (_rowIndex < _rows.Count && _rows[_rowIndex].Refs[0].ChapterNum == _verseRowsRef.ChapterNum) + while ( + _rowIndex < _rows.Count + && _rows[_rowIndex].Refs[0].ChangeVersification(_verseRowsRef.Versification).ChapterNum + == _verseRowsRef.ChapterNum + ) { UpdateUsfmRow row = _rows[_rowIndex]; var ri = new RowInfo(_rowIndex); diff --git a/src/SIL.Machine/Corpora/UsfmToken.cs b/src/SIL.Machine/Corpora/UsfmToken.cs index 43a621b7a..460c5ea26 100644 --- a/src/SIL.Machine/Corpora/UsfmToken.cs +++ b/src/SIL.Machine/Corpora/UsfmToken.cs @@ -180,6 +180,15 @@ public void CopyAttributes(UsfmToken sourceToken) _defaultAttributeName = sourceToken._defaultAttributeName; } + public UsfmToken Copy() + { + UsfmToken copy = new UsfmToken(Type, Marker, Text, EndMarker, Data); + copy.CopyAttributes(this); + copy.LineNumber = LineNumber; + copy.ColumnNumber = ColumnNumber; + return copy; + } + private static void AppendAttribute(List attributes, string name, string value) { value = value?.Trim(); // don't want to have attribute that is just spaces diff --git a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs index cddf0df85..4b793ef54 100644 --- a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs @@ -1,5 +1,6 @@ using System.Collections.Immutable; using NUnit.Framework; +using SIL.Scripture; namespace SIL.Machine.Corpora; @@ -9,7 +10,7 @@ public class UpdateUsfmParserHandlerTests [Test] public void GetUsfm_Verse_CharStyle() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "First verse of the first chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "First verse of the first chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\id MAT - Test\r\n")); @@ -33,8 +34,8 @@ public void GetUsfm_StripAllText() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), - new UpdateUsfmRow(ScrRef("MAT 1:3"), "Update 3"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "Update 3"), ]; string usfm = @"\id MAT - Test @@ -102,9 +103,9 @@ public void GetUsfm_StripParagraphs_PreserveParagraphStyles() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:0/1:rem"), "New remark"), - new UpdateUsfmRow(ScrRef("MAT 1:0/3:ip"), "Another new remark"), - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:0/1:rem"]), "New remark"), + new UpdateUsfmRow(ScrRef(["MAT 1:0/3:ip"]), "Another new remark"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), ]; string usfm = @"\id MAT @@ -156,8 +157,8 @@ public void GetUsfm_PreserveParagraphs() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:0/1:rem"), "Update remark"), - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:0/1:rem"]), "Update remark"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), ]; string usfm = @"\id MAT @@ -201,7 +202,7 @@ public void GetUsfm_PreserveParagraphs() [Test] public void GetUsfm_ParagraphInVerse() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -247,8 +248,8 @@ public void GetUsfm_PreferExisting() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "Update 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Update 2"), ]; string usfm = @"\id MAT - Test @@ -273,8 +274,8 @@ public void GetUsfm_PreferRows() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:6"), "Text 6"), - new UpdateUsfmRow(ScrRef("MAT 1:7"), "Text 7"), + new UpdateUsfmRow(ScrRef(["MAT 1:6"]), "Text 6"), + new UpdateUsfmRow(ScrRef(["MAT 1:7"]), "Text 7"), ]; string target = UpdateUsfm(rows, textBehavior: UpdateUsfmTextBehavior.PreferNew); Assert.That(target, Contains.Substring("\\id MAT - Test\r\n")); @@ -285,7 +286,7 @@ public void GetUsfm_PreferRows() [Test] public void GetUsfm_Verse_StripNote() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 2:1"), "First verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:1"]), "First verse of the second chapter.")]; string target = UpdateUsfm(rows, embedBehavior: UpdateUsfmMarkerBehavior.Strip); Assert.That(target, Contains.Substring("\\v 1 First verse of the second chapter.\r\n")); @@ -294,7 +295,7 @@ public void GetUsfm_Verse_StripNote() [Test] public void GetUsfm_Verse_ReplaceWithNote() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "updated text")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "updated text")]; string usfm = @"\id MAT - Test \c 1 @@ -312,7 +313,7 @@ public void GetUsfm_Verse_ReplaceWithNote() [Test] public void GetUsfm_Verse_RowVerseSegment() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 2:1a"), "First verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:1a"]), "First verse of the second chapter.")]; string target = UpdateUsfm(rows); Assert.That( @@ -326,7 +327,7 @@ public void GetUsfm_Verse_RowVerseSegment() [Test] public void GetUsfm_Verse_UsfmVerseSegment() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 2:7"), "Seventh verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:7"]), "Seventh verse of the second chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\v 7a Seventh verse of the second chapter.\r\n")); @@ -335,7 +336,7 @@ public void GetUsfm_Verse_UsfmVerseSegment() [Test] public void GetUsfm_Verse_MultipleParas() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:2"), "Second verse of the first chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Second verse of the first chapter.")]; string target = UpdateUsfm(rows); Assert.That( @@ -349,7 +350,7 @@ public void GetUsfm_Verse_MultipleParas() [Test] public void GetUsfm_Verse_Table() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 2:9"), "Ninth verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:9"]), "Ninth verse of the second chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\v 9 Ninth verse of the second chapter. \\tcr2 \\tc3 \\tcr4\r\n")); @@ -361,7 +362,7 @@ public void GetUsfm_Verse_RangeSingleRowMultipleVerses() List rows = [ new UpdateUsfmRow( - ScrRef("MAT 2:11", "MAT 2:12"), + ScrRef(["MAT 2:11", "MAT 2:12"]), "Eleventh verse of the second chapter. Twelfth verse of the second chapter." ), ]; @@ -378,7 +379,7 @@ public void GetUsfm_Verse_RangeSingleRowMultipleVerses() [Test] public void GetUsfm_Verse_RangeSingleRowSingleVerse() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 2:11"), "Eleventh verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:11"]), "Eleventh verse of the second chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\v 11-12 Eleventh verse of the second chapter.\r\n")); @@ -389,8 +390,8 @@ public void GetUsfm_Verse_RangeMultipleRowsSingleVerse() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 2:11"), "Eleventh verse of the second chapter."), - new UpdateUsfmRow(ScrRef("MAT 2:12"), "Twelfth verse of the second chapter."), + new UpdateUsfmRow(ScrRef(["MAT 2:11"]), "Eleventh verse of the second chapter."), + new UpdateUsfmRow(ScrRef(["MAT 2:12"]), "Twelfth verse of the second chapter."), ]; string target = UpdateUsfm(rows); @@ -407,9 +408,9 @@ public void GetUsfm_MergeVerseSegments() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 2:2"), "Verse 2."), - new UpdateUsfmRow(ScrRef("MAT 2:2a"), "Verse 2a."), - new UpdateUsfmRow(ScrRef("MAT 2:2b"), "Verse 2b."), + new UpdateUsfmRow(ScrRef(["MAT 2:2"]), "Verse 2."), + new UpdateUsfmRow(ScrRef(["MAT 2:2a"]), "Verse 2a."), + new UpdateUsfmRow(ScrRef(["MAT 2:2b"]), "Verse 2b."), ]; string target = UpdateUsfm(rows); @@ -421,8 +422,8 @@ public void GetUsfm_Verse_OptBreak() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 2:2"), "Second verse of the second chapter."), - new UpdateUsfmRow(ScrRef("MAT 2:3"), "Third verse of the second chapter."), + new UpdateUsfmRow(ScrRef(["MAT 2:2"]), "Second verse of the second chapter."), + new UpdateUsfmRow(ScrRef(["MAT 2:3"]), "Third verse of the second chapter."), ]; string target = UpdateUsfm(rows, embedBehavior: UpdateUsfmMarkerBehavior.Strip); @@ -435,7 +436,7 @@ public void GetUsfm_Verse_OptBreak() [Test] public void GetUsfm_Verse_Milestone() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 2:10"), "Tenth verse of the second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:10"]), "Tenth verse of the second chapter.")]; string target = UpdateUsfm(rows); Assert.That( @@ -447,7 +448,7 @@ public void GetUsfm_Verse_Milestone() [Test] public void GetUsfm_Verse_Unmatched() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:3"), "Third verse of the first chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "Third verse of the first chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\v 3 Third verse of the first chapter.\r\n")); @@ -456,7 +457,7 @@ public void GetUsfm_Verse_Unmatched() [Test] public void GetUsfm_NonVerse_CharStyle() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 2:0/3:s1"), "The second chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:0/3:s1"]), "The second chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\s1 The second chapter.\r\n")); @@ -465,7 +466,7 @@ public void GetUsfm_NonVerse_CharStyle() [Test] public void GetUsfm_NonVerse_Paragraph() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:0/8:s"), "The first chapter.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:0/8:s"]), "The first chapter.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\s The first chapter.\r\n")); @@ -476,11 +477,11 @@ public void GetUsfm_NonVerse_Relaxed() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:0/s"), "The first chapter."), - new UpdateUsfmRow(ScrRef("MAT 1:1"), "First verse of the first chapter."), - new UpdateUsfmRow(ScrRef("MAT 2:0/tr/tc1"), "The first cell of the table."), - new UpdateUsfmRow(ScrRef("MAT 2:0/tr/tc2"), "The second cell of the table."), - new UpdateUsfmRow(ScrRef("MAT 2:0/tr/tc1"), "The third cell of the table."), + new UpdateUsfmRow(ScrRef(["MAT 1:0/s"]), "The first chapter."), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "First verse of the first chapter."), + new UpdateUsfmRow(ScrRef(["MAT 2:0/tr/tc1"]), "The first cell of the table."), + new UpdateUsfmRow(ScrRef(["MAT 2:0/tr/tc2"]), "The second cell of the table."), + new UpdateUsfmRow(ScrRef(["MAT 2:0/tr/tc1"]), "The third cell of the table."), ]; string target = UpdateUsfm(rows); @@ -506,7 +507,7 @@ public void GetUsfm_NonVerse_Sidebar() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 2:3/1:esb/1:ms"), "The first paragraph of the sidebar."), + new UpdateUsfmRow(ScrRef(["MAT 2:3/1:esb/1:ms"]), "The first paragraph of the sidebar."), ]; string target = UpdateUsfm(rows); @@ -518,8 +519,8 @@ public void GetUsfm_NonVerse_Table() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 2:0/1:tr/1:tc1"), "The first cell of the table."), - new UpdateUsfmRow(ScrRef("MAT 2:0/2:tr/1:tc1"), "The third cell of the table."), + new UpdateUsfmRow(ScrRef(["MAT 2:0/1:tr/1:tc1"]), "The first cell of the table."), + new UpdateUsfmRow(ScrRef(["MAT 2:0/2:tr/1:tc1"]), "The third cell of the table."), ]; string target = UpdateUsfm(rows); @@ -538,7 +539,7 @@ public void GetUsfm_NonVerse_OptBreak() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 2:3/1:esb/2:p"), "The second paragraph of the sidebar."), + new UpdateUsfmRow(ScrRef(["MAT 2:3/1:esb/2:p"]), "The second paragraph of the sidebar."), ]; string target = UpdateUsfm(rows); @@ -548,7 +549,7 @@ public void GetUsfm_NonVerse_OptBreak() [Test] public void GetUsfm_NonVerse_Milestone() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 2:7a/1:s"), "A new section header.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 2:7a/1:s"]), "A new section header.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\s A new section header. \\ts-s\\*\r\n")); @@ -557,7 +558,7 @@ public void GetUsfm_NonVerse_Milestone() [Test] public void GetUsfm_NonVerse_SkipNote() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:0/3:ip"), "The introductory paragraph.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:0/3:ip"]), "The introductory paragraph.")]; string target = UpdateUsfm(rows, embedBehavior: UpdateUsfmMarkerBehavior.Strip); Assert.That(target, Contains.Substring("\\ip The introductory paragraph.\r\n")); @@ -566,7 +567,7 @@ public void GetUsfm_NonVerse_SkipNote() [Test] public void GetUsfm_NonVerse_ReplaceWithNote() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:0/3:ip"), "The introductory paragraph.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:0/3:ip"]), "The introductory paragraph.")]; string target = UpdateUsfm(rows); Assert.That( @@ -578,7 +579,7 @@ public void GetUsfm_NonVerse_ReplaceWithNote() [Test] public void GetUsfm_Verse_DoubleVaVp() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 3:1"), "Updating later in the book to start.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 3:1"]), "Updating later in the book to start.")]; string target = UpdateUsfm(rows); Assert.That(target, Contains.Substring("\\id MAT - Test\r\n")); @@ -591,7 +592,7 @@ public void GetUsfm_Verse_DoubleVaVp() [Test] public void GetUsfm_Verse_LastSegment() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Updating the last verse.")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Updating the last verse.")]; string usfm = @"\id MAT - Test \c 1 @@ -616,12 +617,12 @@ public void GetUsfm_Verse_UpdateRowsBeforeText() { List rows = [ - new UpdateUsfmRow(ScrRef("GEN 1:1"), "Update rows before the start"), - new UpdateUsfmRow(ScrRef("GEN 1:2"), "Update rows before the start"), - new UpdateUsfmRow(ScrRef("GEN 1:3"), "Update rows before the start"), - new UpdateUsfmRow(ScrRef("GEN 1:4"), "Update rows before the start"), - new UpdateUsfmRow(ScrRef("GEN 1:5"), "Update rows before the start"), - new UpdateUsfmRow(ScrRef("MAT 1:0/3:ip"), "The introductory paragraph."), + new UpdateUsfmRow(ScrRef(["GEN 1:1"]), "Update rows before the start"), + new UpdateUsfmRow(ScrRef(["GEN 1:2"]), "Update rows before the start"), + new UpdateUsfmRow(ScrRef(["GEN 1:3"]), "Update rows before the start"), + new UpdateUsfmRow(ScrRef(["GEN 1:4"]), "Update rows before the start"), + new UpdateUsfmRow(ScrRef(["GEN 1:5"]), "Update rows before the start"), + new UpdateUsfmRow(ScrRef(["MAT 1:0/3:ip"]), "The introductory paragraph."), ]; string target = UpdateUsfm(rows); @@ -636,8 +637,8 @@ public void GetUsfm_StripParagraphs() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:0/2:p"), "Update Paragraph"), - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update Verse 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:0/2:p"]), "Update Paragraph"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update Verse 1"), ]; string usfm = @@ -684,7 +685,7 @@ public void GetUsfm_PreservationRawStrings() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), @"Update all in one row \f \fr 1.1 \ft Some note \f*"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), @"Update all in one row \f \fr 1.1 \ft Some note \f*"), ]; string usfm = @@ -705,7 +706,7 @@ public void GetUsfm_PreservationRawStrings() [Test] public void GetUsfm_BeginningOfVerseEmbed() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Updated text")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Updated text")]; string usfm = @"\id MAT - Test @@ -722,10 +723,236 @@ public void GetUsfm_BeginningOfVerseEmbed() AssertUsfmEquals(target, result); } + [Test] + public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneFewerChapter() + { + List rows = + [ + new UpdateUsfmRow(ScrRef(["MAL 1:1"], ScrVers.Original), "Updated verse 1"), + new UpdateUsfmRow(ScrRef(["MAL 3:24"], ScrVers.Original), "Updated verse 2"), + ]; + string usfm = + @"\id MAL +\c 1 +\v 1-14 +\c 2 +\v 1-17 +\c 3 +\v 1-18 +\c 4 +\v 1-5 +\v 6 +"; + string target = UpdateUsfm(rows, usfm, bookId: "MAL", versification: ScrVers.English); + string result = + @"\id MAL +\c 1 +\v 1-14 Updated verse 1 +\c 2 +\v 1-17 +\c 3 +\v 1-18 +\c 4 +\v 1-5 +\v 6 Updated verse 2 +"; + AssertUsfmEquals(target, result); + + target = UpdateUsfm( + rows, + usfm, + bookId: "MAL", + versification: ScrVers.English, + convertUsfmToUpdateRowVersification: true + ); + result = + @"\id MAL +\c 1 +\v 1-14 Updated verse 1 +\c 2 +\v 1-17 +\c 3 +\v 1-18 +\v 19-23 +\v 24 Updated verse 2 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneMoreChapter() + { + List rows = + [ + new UpdateUsfmRow(ScrRef(["MAL 1:1"], ScrVers.English), "Updated verse 1"), + new UpdateUsfmRow(ScrRef(["MAL 4:6"], ScrVers.English), "Updated verse 2"), + ]; + string usfm = + @"\id MAL +\c 1 +\v 1-14 +\c 2 +\v 1-17 +\c 3 +\v 1-18 +\v 19-23 +\v 24 +"; + + string target = UpdateUsfm(rows, usfm, bookId: "MAL", versification: ScrVers.Original); + string result = + @"\id MAL +\c 1 +\v 1-14 Updated verse 1 +\c 2 +\v 1-17 +\c 3 +\v 1-18 +\v 19-23 +\v 24 Updated verse 2 +"; + AssertUsfmEquals(target, result); + + target = UpdateUsfm( + rows, + usfm, + bookId: "MAL", + versification: ScrVers.Original, + convertUsfmToUpdateRowVersification: true + ); + result = + @"\id MAL +\c 1 +\v 1-14 Updated verse 1 +\c 2 +\v 1-17 +\c 3 +\v 1-18 +\c 4 +\v 1-5 +\v 6 Updated verse 2 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ConvertUsfmToUpdateRowVersification_EmptyChapters() + { + List rows = []; + + string usfm = + @"\id MAT - Test +\c 1 +\c 2 +\c 3 +"; + + string target = UpdateUsfm(rows, usfm, convertUsfmToUpdateRowVersification: true); + string result = + @"\id MAT - Test +\c 1 +\c 2 +\c 3 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneFewerBook() + { + // Russian Orthodox vs. Original + // PSA 151:1-7 = PS2 1:1-7 + + List rows = + [ + new UpdateUsfmRow(ScrRef(["PSA 150:1"], ScrVers.Original), "Updated verse 1"), + new UpdateUsfmRow(ScrRef(["PS2 1:1"], ScrVers.Original), "Updated verse 2"), + ]; + + string usfm = + @"\id PSA - Test +\c 150 +\v 1-6 +\c 151 +\v 1-7 +"; + + string target = UpdateUsfm( + rows, + usfm, + convertUsfmToUpdateRowVersification: false, + versification: ScrVers.RussianOrthodox + ); + string result = + @"\id PSA - Test +\c 150 +\v 1-6 Updated verse 1 +\c 151 +\v 1-7 Updated verse 2 +"; + AssertUsfmEquals(target, result); + + target = UpdateUsfm( + rows, + usfm, + convertUsfmToUpdateRowVersification: true, + versification: ScrVers.RussianOrthodox + ); + result = + @"\id PSA - Test +\c 150 +\v 1-6 Updated verse 1 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneMoreBook() + { + // Russian Orthodox vs. Original + // DAN 3:24-90 = DAG 3:24-90 + // DAN 3:91-100 = DAN 3:24-33 + + // Original + // S3Y 1:1-29 = DAG 3:24-52 + + List rows = + [ + new UpdateUsfmRow(ScrRef(["DAN 3:1"], ScrVers.RussianOrthodox), "Updated verse 1"), + // This row will map to another book DAG so it will not appear in the updated DAN + new UpdateUsfmRow(ScrRef(["DAN 3:24"], ScrVers.RussianOrthodox), "Updated verse 2"), + ]; + + string usfm = + @"\id DAN - Test +\c 1 +\c 2 +\c 3 +\v 1-23 +\c 4 +"; + + string target = UpdateUsfm( + rows, + usfm, + convertUsfmToUpdateRowVersification: true, + versification: ScrVers.Original + ); + string result = + @"\id DAN - Test +\c 1 +\c 2 +\c 3 +\v 1-23 Updated verse 1 +\c 4 +"; + AssertUsfmEquals(target, result); + } + [Test] public void CrossReferenceDontUpdate() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1/1:x"), "Update the cross reference")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1/1:x"]), "Update the cross reference")]; string usfm = @"\id MAT - Test \c 1 @@ -743,7 +970,7 @@ public void CrossReferenceDontUpdate() [Test] public void PreserveFig() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update")]; string usfm = @"\id MAT - Test \c 1 @@ -763,8 +990,8 @@ public void NoteExplicitEndMarkers() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update text"), - new UpdateUsfmRow(ScrRef("MAT 1:1/1:f"), "Update note"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update text"), + new UpdateUsfmRow(ScrRef(["MAT 1:1/1:f"]), "Update note"), ]; string usfm = @"\id MAT - Test @@ -791,7 +1018,7 @@ public void NoteExplicitEndMarkers() [Test] public void UpdateBlock_Verse_PreserveParas() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -821,7 +1048,7 @@ public void UpdateBlock_Verse_PreserveParas() [Test] public void UpdateBlock_Verse_StripParas() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -851,7 +1078,7 @@ public void UpdateBlock_Verse_StripParas() [Test] public void UpdateBlock_Verse_Range() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -879,7 +1106,7 @@ public void UpdateBlock_Verse_Range() [Test] public void UpdateBlock_Verse_Range_RightToLeftMarker() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1", "MAT 1:2", "MAT 1:3"), "Update 1-3")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1", "MAT 1:2", "MAT 1:3"]), "Update 1-3")]; string usfm = @"\id MAT - Test \c 1 @@ -910,7 +1137,7 @@ public void UpdateBlock_Verse_Range_RightToLeftMarker() [Test] public void UpdateBlock_Footnote_PreserveEmbeds() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -940,7 +1167,7 @@ public void UpdateBlock_Footnote_PreserveEmbeds() [Test] public void UpdateBlock_Footnote_StripEmbeds() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -970,7 +1197,7 @@ public void UpdateBlock_Footnote_StripEmbeds() [Test] public void UpdateBlock_NonVerse() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:0/1:s"), "Updated section Header")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:0/1:s"]), "Updated section Header")]; string usfm = @"\id MAT - Test \s Section header @@ -994,7 +1221,7 @@ public void UpdateBlock_NonVerse() [Test] public void UpdateBlock_Verse_PreserveStyles() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1026,7 +1253,7 @@ public void UpdateBlock_Verse_PreserveStyles() [Test] public void UpdateBlock_Verse_StripStyles() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1058,7 +1285,7 @@ public void UpdateBlock_Verse_StripStyles() [Test] public void UpdateBlock_Verse_SectionHeader() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1096,7 +1323,7 @@ public void UpdateBlock_Verse_SectionHeader() [Test] public void UpdateBlock_Verse_SectionHeaderInVerse() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1129,7 +1356,7 @@ public void UpdateBlock_Verse_SectionHeaderInVerse() [Test] public void UpdateBlock_NonVerse_ParagraphEndOfVerse() { - List rows = [new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1")]; string usfm = @"\id MAT - Test \c 1 @@ -1160,11 +1387,11 @@ public void GetUsfm_HeaderReferenceParagraphs() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "new verse 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "new verse 2"), - new UpdateUsfmRow(ScrRef("MAT 1:3"), "new verse 3"), - new UpdateUsfmRow(ScrRef("MAT 2:1"), "new verse 1"), - new UpdateUsfmRow(ScrRef("MAT 2:2"), "new verse 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "new verse 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "new verse 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "new verse 3"), + new UpdateUsfmRow(ScrRef(["MAT 2:1"]), "new verse 1"), + new UpdateUsfmRow(ScrRef(["MAT 2:2"]), "new verse 2"), ]; string usfm = @@ -1215,16 +1442,16 @@ public void GetUsfm_OutOfOrderVerses() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "new verse 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "new verse 2"), - new UpdateUsfmRow(ScrRef("MAT 1:3"), "new verse 3"), - new UpdateUsfmRow(ScrRef("MAT 1:4"), "new verse 4"), - new UpdateUsfmRow(ScrRef("MAT 1:5"), "new verse 5"), - new UpdateUsfmRow(ScrRef("MAT 1:6a"), "new verse 6a"), - new UpdateUsfmRow(ScrRef("MAT 1:6b"), "new verse 6b"), - new UpdateUsfmRow(ScrRef("MAT 1:6b/1:s"), "new section"), - new UpdateUsfmRow(ScrRef("MAT 1:7"), "new verse 7"), - new UpdateUsfmRow(ScrRef("MAT 1:8"), "new verse 8"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "new verse 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "new verse 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "new verse 3"), + new UpdateUsfmRow(ScrRef(["MAT 1:4"]), "new verse 4"), + new UpdateUsfmRow(ScrRef(["MAT 1:5"]), "new verse 5"), + new UpdateUsfmRow(ScrRef(["MAT 1:6a"]), "new verse 6a"), + new UpdateUsfmRow(ScrRef(["MAT 1:6b"]), "new verse 6b"), + new UpdateUsfmRow(ScrRef(["MAT 1:6b/1:s"]), "new section"), + new UpdateUsfmRow(ScrRef(["MAT 1:7"]), "new verse 7"), + new UpdateUsfmRow(ScrRef(["MAT 1:8"]), "new verse 8"), ]; string usfm = @@ -1274,10 +1501,10 @@ public void GetUsfm_DuplicateVerses() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "new verse 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "new verse 2"), - new UpdateUsfmRow(ScrRef("MAT 1:3"), "new verse 3"), - new UpdateUsfmRow(ScrRef("MAT 1:4"), "new verse 4"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "new verse 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "new verse 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "new verse 3"), + new UpdateUsfmRow(ScrRef(["MAT 1:4"]), "new verse 4"), ]; string usfm = @@ -1312,11 +1539,11 @@ public void GetUsfm_IdTags() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:0/1:s"), "new section header"), - new UpdateUsfmRow(ScrRef("MAT 1:1"), "new verse 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "new verse 2"), - new UpdateUsfmRow(ScrRef("MAT 1:3"), "new verse 3"), - new UpdateUsfmRow(ScrRef("MAT 1:4"), "new verse 4"), + new UpdateUsfmRow(ScrRef(["MAT 1:0/1:s"]), "new section header"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "new verse 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "new verse 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:3"]), "new verse 3"), + new UpdateUsfmRow(ScrRef(["MAT 1:4"]), "new verse 4"), ]; string usfm = @@ -1357,8 +1584,8 @@ public void GetUsfm_PassRemark() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "Update 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Update 2"), ]; string usfm = @@ -1412,8 +1639,8 @@ public void GetUsfm_PassRemark_NoBodyParagraph() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "Update 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Update 2"), ]; string usfm = @@ -1465,8 +1692,8 @@ public void GetUsfm_PassRemark0_NoExistingRemark() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "Update 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Update 2"), ]; string usfm = @@ -1503,8 +1730,8 @@ public void GetUsfm_MultipleRemarksSameChapter() { List rows = [ - new UpdateUsfmRow(ScrRef("MAT 1:1"), "Update 1"), - new UpdateUsfmRow(ScrRef("MAT 1:2"), "Update 2"), + new UpdateUsfmRow(ScrRef(["MAT 1:1"]), "Update 1"), + new UpdateUsfmRow(ScrRef(["MAT 1:2"]), "Update 2"), ]; string usfm = @@ -1546,7 +1773,7 @@ public void GetUsfm_MultipleRemarksSameChapter() [Test] public void UpdateBlock_FootnoteInPublishedChapterNumber() { - List rows = [new UpdateUsfmRow(ScrRef("ESG 1:0/2:s"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["ESG 1:0/2:s"]), "Update 1")]; string usfm = @"\id ESG - Test \c 1 @@ -1589,7 +1816,7 @@ public void UpdateBlock_FootnoteInPublishedChapterNumber() [Test] public void UpdateBlock_FootnoteAtStartOfChapterWithPrecedingText() { - List rows = [new UpdateUsfmRow(ScrRef("ESG 1:0/2:s"), "Update 1")]; + List rows = [new UpdateUsfmRow(ScrRef(["ESG 1:0/2:s"]), "Update 1")]; string usfm = @"\id ESG - Test \c 1 @@ -1719,7 +1946,8 @@ public void FilterChapters_WithBadChapterReference() AssertUsfmEquals(target, result); } - private static ScriptureRef[] ScrRef(params string[] refs) => [.. refs.Select(r => ScriptureRef.Parse(r))]; + private static ScriptureRef[] ScrRef(IEnumerable refs, ScrVers? versification = null) => + [.. refs.Select(r => ScriptureRef.Parse(r, versification))]; private static string UpdateUsfm( IReadOnlyList? rows = null, @@ -1733,15 +1961,18 @@ private static string UpdateUsfm( IEnumerable? preserveParagraphStyles = null, IEnumerable? usfmUpdateBlockHandlers = null, IEnumerable<(int, string)>? remarks = null, - bool compareSegments = false + bool compareSegments = false, + bool convertUsfmToUpdateRowVersification = false, + string? bookId = null, + ScrVers? versification = null ) { - const string BookId = "MAT"; + bookId ??= "MAT"; if (source is null) { var updater = new FileParatextProjectTextUpdater(CorporaTestHelpers.UsfmTestProjectPath); return updater.UpdateUsfm( - BookId, + bookId, rows, chapters, idText, @@ -1753,17 +1984,23 @@ private static string UpdateUsfm( usfmUpdateBlockHandlers, remarks, (_) => false, - compareSegments + compareSegments, + convertUsfmToUpdateRowVersification ); } else { source = source.Trim().ReplaceLineEndings("\r\n") + "\r\n"; - var settings = new DefaultParatextProjectSettings(fileNameForm: BookId, fileNameSuffix: string.Empty); - var files = new Dictionary { [BookId] = source }; + var settings = new DefaultParatextProjectSettings( + fileNameForm: "MAT", + fileNamePrefix: string.Empty, + fileNameSuffix: string.Empty, + versification: versification + ); + var files = new Dictionary { [bookId] = source }; var updater = new MemoryParatextProjectTextUpdater(files, settings); return updater.UpdateUsfm( - BookId, + bookId, rows, chapters, idText, @@ -1775,7 +2012,8 @@ private static string UpdateUsfm( usfmUpdateBlockHandlers, remarks, (_) => false, - compareSegments + compareSegments, + convertUsfmToUpdateRowVersification ); } } From a6fe14bd4e47d6ea2371f7378e9ebca5a3fc5cbe Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Wed, 12 Aug 2026 21:38:47 -0400 Subject: [PATCH 2/3] Address reviewer comments --- .../Corpora/UpdateUsfmParserHandler.cs | 49 ++--------- src/SIL.Machine/Corpora/UsfmToken.cs | 5 +- .../Corpora/UpdateUsfmParserHandlerTests.cs | 87 ++++++++++++++++++- 3 files changed, 95 insertions(+), 46 deletions(-) diff --git a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs index 6672eede6..c788c4648 100644 --- a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs +++ b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs @@ -67,7 +67,6 @@ public class UpdateUsfmParserHandler : ScriptureRefUsfmParserHandlerBase private readonly Func _errorHandler; private readonly bool _compareSegments; private readonly bool _convertUsfmToUpdateRowVersification; - private UsfmToken _currentChapterToken; private int _currentChapterNum; private bool _skipNextVerseText; @@ -122,7 +121,6 @@ public UpdateUsfmParserHandler( _compareSegments = compareSegments; _convertUsfmToUpdateRowVersification = convertUsfmToUpdateRowVersification && _updateRowsVersification != _usfmVersification; - _currentChapterToken = null; _currentChapterNum = 0; _skipNextVerseText = false; } @@ -132,11 +130,6 @@ public UpdateUsfmParserHandler( public override void EndUsfm(UsfmParserState state) { CollectUpdatableTokens(state); - if (_currentChapterToken != null) - { - _tokens.Add(_currentChapterToken); - _currentChapterToken = null; - } base.EndUsfm(state); } @@ -166,12 +159,6 @@ public override void EndBook(UsfmParserState state, string marker) UsfmUpdateBlock updateBlock = _updateBlocks.Pop(); _tokens.AddRange(updateBlock.GetTokens()); - if (_currentChapterToken != null) - { - _tokens.Add(_currentChapterToken); - _currentChapterToken = null; - } - base.EndBook(state, marker); } @@ -632,28 +619,16 @@ private VerseRef UpdateVerseData(UsfmParserState state, UsfmToken token) verseRef = verseRef.ChangeVersificationWithSegments(_updateRowsVersification); if (verseRef.ChapterNum != _currentChapterNum && state.VerseRef.BookNum == verseRef.BookNum) { - if (_currentChapterToken != null) - { - _tokens.Add(_currentChapterToken.Copy()); - _currentChapterToken = null; - } - else - { - UsfmToken newChapterToken = new UsfmToken( - UsfmTokenType.Chapter, - "c", - "", - "", - verseRef.ChapterNum.ToString() - ); - _tokens.Add(newChapterToken); - } + UsfmToken newChapterToken = new UsfmToken( + UsfmTokenType.Chapter, + "c", + "", + "", + verseRef.ChapterNum.ToString() + ); + _tokens.Add(newChapterToken); _currentChapterNum = verseRef.ChapterNum; } - if (verseRef.ChapterNum == verseRef.Versification.GetLastChapter(verseRef.BookNum)) - { - _currentChapterToken = null; - } } return verseRef; @@ -670,13 +645,7 @@ private void CollectReadonlyTokens(UsfmParserState state) } else { - if (_convertUsfmToUpdateRowVersification && token.Type == UsfmTokenType.Chapter) - { - if (_currentChapterToken != null) - _tokens.Add(_currentChapterToken); - _currentChapterToken = token; - } - else + if (!_convertUsfmToUpdateRowVersification || token.Type != UsfmTokenType.Chapter) { _tokens.Add(token); } diff --git a/src/SIL.Machine/Corpora/UsfmToken.cs b/src/SIL.Machine/Corpora/UsfmToken.cs index 460c5ea26..28e4765ad 100644 --- a/src/SIL.Machine/Corpora/UsfmToken.cs +++ b/src/SIL.Machine/Corpora/UsfmToken.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; +using SIL.ObjectModel; namespace SIL.Machine.Corpora { @@ -22,7 +23,7 @@ public enum UsfmTokenType Unknown, } - public class UsfmToken : IEquatable + public class UsfmToken : IEquatable, ICloneable { private const string FullAttributeStr = @"(?[-\w]+)\s*\=\s*\""(?.+?)\""\s*"; private static readonly Regex AttributeRegex = new Regex( @@ -180,7 +181,7 @@ public void CopyAttributes(UsfmToken sourceToken) _defaultAttributeName = sourceToken._defaultAttributeName; } - public UsfmToken Copy() + public UsfmToken Clone() { UsfmToken copy = new UsfmToken(Type, Marker, Text, EndMarker, Data); copy.CopyAttributes(this); diff --git a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs index 4b793ef54..ee74a4fb9 100644 --- a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs @@ -880,6 +880,7 @@ public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneFewerBook() string target = UpdateUsfm( rows, usfm, + bookId: "PSA", convertUsfmToUpdateRowVersification: false, versification: ScrVers.RussianOrthodox ); @@ -895,6 +896,7 @@ public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneFewerBook() target = UpdateUsfm( rows, usfm, + bookId: "PSA", convertUsfmToUpdateRowVersification: true, versification: ScrVers.RussianOrthodox ); @@ -925,26 +927,103 @@ public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneMoreBook() string usfm = @"\id DAN - Test -\c 1 -\c 2 \c 3 \v 1-23 \c 4 +\v 1 "; string target = UpdateUsfm( rows, usfm, + bookId: "DAN", convertUsfmToUpdateRowVersification: true, versification: ScrVers.Original ); string result = @"\id DAN - Test -\c 1 -\c 2 \c 3 \v 1-23 Updated verse 1 \c 4 +\v 1 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ConvertUsfmToUpdateRowVersification_BackOneVerseToPreviousChapter() + { + // English vs. Original + // ISA 9:1 = ISA 8:23 + + List rows = + [ + new UpdateUsfmRow(ScrRef(["ISA 9:1"], ScrVers.English), "Updated verse 1"), + new UpdateUsfmRow(ScrRef(["ISA 9:2"], ScrVers.English), "Updated verse 2"), + ]; + + string usfm = + @"\id ISA - Test +\c 8 +\v 22 +\v 23 +\c 9 +\v 1 +"; + + string target = UpdateUsfm( + rows, + usfm, + bookId: "ISA", + convertUsfmToUpdateRowVersification: true, + versification: ScrVers.Original + ); + string result = + @"\id ISA - Test +\c 8 +\v 22 +\c 9 +\v 1 Updated verse 1 +\v 2 Updated verse 2 +"; + AssertUsfmEquals(target, result); + } + + [Test] + public void GetUsfm_ConvertUsfmToUpdateRowVersification_ForwardOneVerseToNextChapter() + { + // Original vs. English + // ISA 8:23 = ISA 9:1 + + List rows = + [ + new UpdateUsfmRow(ScrRef(["ISA 8:23"], ScrVers.Original), "Updated verse 23"), + new UpdateUsfmRow(ScrRef(["ISA 9:1"], ScrVers.Original), "Updated verse 1"), + ]; + + string usfm = + @"\id ISA - Test +\c 8 +\v 22 +\c 9 +\v 1 +\v 2 +"; + + string target = UpdateUsfm( + rows, + usfm, + bookId: "ISA", + convertUsfmToUpdateRowVersification: true, + versification: ScrVers.English + ); + string result = + @"\id ISA - Test +\c 8 +\v 22 +\v 23 Updated verse 23 +\c 9 +\v 1 Updated verse 1 "; AssertUsfmEquals(target, result); } From 29d4f7fa4c4efa2ee2834801bc20c02f6df78738 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Wed, 12 Aug 2026 22:28:28 -0400 Subject: [PATCH 3/3] Remove empty chapter test --- .../Corpora/UpdateUsfmParserHandlerTests.cs | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs index ee74a4fb9..254e0f752 100644 --- a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs @@ -835,28 +835,6 @@ public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneMoreChapter() AssertUsfmEquals(target, result); } - [Test] - public void GetUsfm_ConvertUsfmToUpdateRowVersification_EmptyChapters() - { - List rows = []; - - string usfm = - @"\id MAT - Test -\c 1 -\c 2 -\c 3 -"; - - string target = UpdateUsfm(rows, usfm, convertUsfmToUpdateRowVersification: true); - string result = - @"\id MAT - Test -\c 1 -\c 2 -\c 3 -"; - AssertUsfmEquals(target, result); - } - [Test] public void GetUsfm_ConvertUsfmToUpdateRowVersification_OneFewerBook() {