Skip to content

Convert USFM versification #472

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Enkidu93 wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Convert USFM versification #472

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9aa1317
Add ability to convert usfm when updating to the versification of the…
Enkidu93 Aug 12, 2026
a6fe14b
Address reviewer comments
Enkidu93 Aug 13, 2026
29d4f7f
Remove empty chapter test
Enkidu93 Aug 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ public string UpdateUsfm(
IEnumerable<IUsfmUpdateBlockHandler> updateBlockHandlers = null,
IEnumerable<(int, string)> remarks = null,
Func<UsfmUpdateBlockHandlerException, bool> errorHandler = null,
bool compareSegments = false
bool compareSegments = false,
bool convertUsfmToUpdateRowVersification = false
)
{
string fileName = _settings.GetBookFileName(bookId);
Expand All @@ -57,7 +58,9 @@ public string UpdateUsfm(
updateBlockHandlers,
remarks,
errorHandler,
compareSegments
compareSegments,
_settings.Versification,
convertUsfmToUpdateRowVersification
);
try
{
Expand Down
78 changes: 68 additions & 10 deletions src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ public class UpdateUsfmParserHandler : ScriptureRefUsfmParserHandlerBase
private int _verseRowIndex;
private readonly Dictionary<VerseRef, List<RowInfo>> _verseRowsMap;
private readonly ScrVers _updateRowsVersification;
private readonly ScrVers _usfmVersification;
private readonly List<UsfmToken> _tokens;
private readonly List<UsfmToken> _updatedText;
private readonly List<UsfmToken> _embedTokens;
Expand All @@ -65,6 +66,9 @@ public class UpdateUsfmParserHandler : ScriptureRefUsfmParserHandlerBase
private int _tokenIndex;
private readonly Func<UsfmUpdateBlockHandlerException, bool> _errorHandler;
private readonly bool _compareSegments;
private readonly bool _convertUsfmToUpdateRowVersification;
private int _currentChapterNum;
private bool _skipNextVerseText;

/// <param name="rows">UpdateUsfmRows must be in order</param>
public UpdateUsfmParserHandler(
Expand All @@ -78,7 +82,9 @@ public UpdateUsfmParserHandler(
IEnumerable<IUsfmUpdateBlockHandler> updateBlockHandlers = null,
IEnumerable<(int, string)> remarks = null,
Func<UsfmUpdateBlockHandlerException, bool> errorHandler = null,
bool compareSegments = false
bool compareSegments = false,
ScrVers usfmVersification = null,
bool convertUsfmToUpdateRowVersification = false
)
{
_rows = rows ?? Array.Empty<UpdateUsfmRow>();
Expand All @@ -89,6 +95,7 @@ public UpdateUsfmParserHandler(
_updateRowsVersification = ScrVers.English;
if (_rows.Count > 0)
_updateRowsVersification = _rows.First(r => r.Refs.Count > 0).Refs[0].Versification;
_usfmVersification = usfmVersification ?? _updateRowsVersification;
_tokens = new List<UsfmToken>();
_updatedText = new List<UsfmToken>();
_updateBlocks = new Stack<UsfmUpdateBlock>();
Expand All @@ -112,6 +119,10 @@ public UpdateUsfmParserHandler(
if (_errorHandler == null)
_errorHandler = (error) => false;
_compareSegments = compareSegments;
_convertUsfmToUpdateRowVersification =
convertUsfmToUpdateRowVersification && _updateRowsVersification != _usfmVersification;
_currentChapterNum = 0;
_skipNextVerseText = false;
}

public IReadOnlyList<UsfmToken> Tokens => _tokens;
Expand Down Expand Up @@ -398,7 +409,11 @@ protected override void StartVerseText(UsfmParserState state, IReadOnlyList<Scri

protected override void EndVerseText(UsfmParserState state, IReadOnlyList<ScriptureRef> scriptureRefs)
{
EndUpdateBlock(state, scriptureRefs);
if (!_skipNextVerseText)
{
EndUpdateBlock(state, scriptureRefs);
_skipNextVerseText = false;
}
}

protected override void StartNonVerseText(UsfmParserState state, ScriptureRef scriptureRef)
Expand Down Expand Up @@ -564,9 +579,16 @@ private void CollectUpdatableTokens(UsfmParserState state)
UsfmToken token = state.Tokens[_tokenIndex];
if (token.Type == UsfmTokenType.Verse)
{
string sanitizedVerseData = SanitizeVerseData(token.Data);
token = new UsfmToken(token.Type, token.Marker, token.Text, token.EndMarker, sanitizedVerseData);
VerseRef updatedVerse = UpdateVerseData(state, token);
if (updatedVerse.BookNum != state.VerseRef.BookNum)
{
_tokenIndex++;
_skipNextVerseText = true;
continue;
}
token = new UsfmToken(token.Type, token.Marker, token.Text, token.EndMarker, updatedVerse.Verse);
}

if (CurrentTextType == ScriptureTextType.Embed)
{
_embedTokens.Add(token);
Expand All @@ -582,10 +604,36 @@ private void CollectUpdatableTokens(UsfmParserState state)
{
_tokens.Add(token);
}

_tokenIndex++;
}
}

private VerseRef UpdateVerseData(UsfmParserState state, UsfmToken token)
{
string updatedVerseData = SanitizeVerseData(token.Data);
VerseRef verseRef = state.VerseRef;
verseRef.Verse = updatedVerseData;
if (_convertUsfmToUpdateRowVersification)
{
verseRef = verseRef.ChangeVersificationWithSegments(_updateRowsVersification);
if (verseRef.ChapterNum != _currentChapterNum && state.VerseRef.BookNum == verseRef.BookNum)
{
UsfmToken newChapterToken = new UsfmToken(
UsfmTokenType.Chapter,
"c",
"",
"",
verseRef.ChapterNum.ToString()
);
_tokens.Add(newChapterToken);
_currentChapterNum = verseRef.ChapterNum;
}
}

return verseRef;
}

private void CollectReadonlyTokens(UsfmParserState state)
{
while (_tokenIndex <= state.Index + state.SpecialTokenCount)
Expand All @@ -597,7 +645,10 @@ private void CollectReadonlyTokens(UsfmParserState state)
}
else
{
_tokens.Add(token);
if (!_convertUsfmToUpdateRowVersification || token.Type != UsfmTokenType.Chapter)
{
_tokens.Add(token);
}
}
_tokenIndex++;
}
Expand Down Expand Up @@ -663,10 +714,13 @@ private bool HasNewText()
private void StartUpdateBlock(IReadOnlyList<ScriptureRef> scriptureRefs)
{
(IReadOnlyList<string> rowTexts, Dictionary<string, object> metadata) = AdvanceRows(scriptureRefs);
_updateBlocks.Push(
new UsfmUpdateBlock(scriptureRefs, metadata: metadata ?? new Dictionary<string, object>())
);
PushUpdatedText(rowTexts.Select(t => new UsfmToken(t + " ")));
if (!_skipNextVerseText)
{
_updateBlocks.Push(
new UsfmUpdateBlock(scriptureRefs, metadata: metadata ?? new Dictionary<string, object>())
);
PushUpdatedText(rowTexts.Select(t => new UsfmToken(t + " ")));
}
}

private void EndUpdateBlock(UsfmParserState state, IReadOnlyList<ScriptureRef> scriptureRefs)
Expand Down Expand Up @@ -757,7 +811,11 @@ private bool IsNonverseParagraph(UsfmParserState state, UsfmUpdateBlockElement e
private void UpdateVerseRowsMap()
{
_verseRowsMap.Clear();
while (_rowIndex < _rows.Count && _rows[_rowIndex].Refs[0].ChapterNum == _verseRowsRef.ChapterNum)
while (
_rowIndex < _rows.Count
&& _rows[_rowIndex].Refs[0].ChangeVersification(_verseRowsRef.Versification).ChapterNum
== _verseRowsRef.ChapterNum
)
{
UpdateUsfmRow row = _rows[_rowIndex];
var ri = new RowInfo(_rowIndex);
Expand Down
12 changes: 11 additions & 1 deletion src/SIL.Machine/Corpora/UsfmToken.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using SIL.ObjectModel;

namespace SIL.Machine.Corpora
{
Expand All @@ -22,7 +23,7 @@ public enum UsfmTokenType
Unknown,
}

public class UsfmToken : IEquatable<UsfmToken>
public class UsfmToken : IEquatable<UsfmToken>, ICloneable<UsfmToken>
{
private const string FullAttributeStr = @"(?<name>[-\w]+)\s*\=\s*\""(?<value>.+?)\""\s*";
private static readonly Regex AttributeRegex = new Regex(
Expand Down Expand Up @@ -180,6 +181,15 @@ public void CopyAttributes(UsfmToken sourceToken)
_defaultAttributeName = sourceToken._defaultAttributeName;
}

public UsfmToken Clone()
{
UsfmToken copy = new UsfmToken(Type, Marker, Text, EndMarker, Data);
copy.CopyAttributes(this);
copy.LineNumber = LineNumber;
copy.ColumnNumber = ColumnNumber;
return copy;
}

private static void AppendAttribute(List<UsfmAttribute> attributes, string name, string value)
{
value = value?.Trim(); // don't want to have attribute that is just spaces
Expand Down
Loading
Loading