From 21f384a15cf7b6e5996814b5dd8a9a812aa50017 Mon Sep 17 00:00:00 2001 From: Ken Perry Date: Sun, 18 Oct 2026 01:56:41 -0400 Subject: [PATCH] Keep Word page breaks when importing DOCX Pandoc drops Word page breaks, so imported documents lost them. Before pandoc runs, DocxPageBreaks replaces each manual page break and each paragraph with "Page break before" turned on with marker text. The new docx-page-break.lua filter splits paragraphs at the marker, bbx.lua writes it as bb:pageBreak, and FixPageBreak turns it into newPagesBefore on the next block, the same option the Page Break tool sets. A table gets the option on the table and its braille copy. --- .../main/dist/programData/pandoc/lua/bbx.lua | 4 + .../pandoc/lua/docx-page-break.lua | 136 ++++++++++++ .../archiver2/PandocArchiverLoader.kt | 14 +- .../brailleblaster/pandoc/DocxPageBreaks.kt | 71 ++++++ .../org/brailleblaster/pandoc/FixPageBreak.kt | 76 +++++++ .../pandoc/PandocDocxPageBreakTest.kt | 209 ++++++++++++++++++ 6 files changed, 506 insertions(+), 4 deletions(-) create mode 100644 brailleblaster-core/src/main/dist/programData/pandoc/lua/docx-page-break.lua create mode 100644 brailleblaster-core/src/main/java/org/brailleblaster/pandoc/DocxPageBreaks.kt create mode 100644 brailleblaster-core/src/main/java/org/brailleblaster/pandoc/FixPageBreak.kt create mode 100644 brailleblaster-core/src/test/java/org/brailleblaster/pandoc/PandocDocxPageBreakTest.kt diff --git a/brailleblaster-core/src/main/dist/programData/pandoc/lua/bbx.lua b/brailleblaster-core/src/main/dist/programData/pandoc/lua/bbx.lua index f8a3d3da..eca45096 100644 --- a/brailleblaster-core/src/main/dist/programData/pandoc/lua/bbx.lua +++ b/brailleblaster-core/src/main/dist/programData/pandoc/lua/bbx.lua @@ -724,6 +724,10 @@ end function Div(s, attr) local customStyle = getCustomStyle(attr) + -- From docx-page-break.lua; FixPageBreak moves it onto the next block. + if customStyle == "BB Page Break" then + return '' + end if isListParagraphStyle(customStyle) or isPlainListStyle(customStyle) then local item = listItemText(s) if string.len(string.gsub(item, '%s+', '')) == 0 then diff --git a/brailleblaster-core/src/main/dist/programData/pandoc/lua/docx-page-break.lua b/brailleblaster-core/src/main/dist/programData/pandoc/lua/docx-page-break.lua new file mode 100644 index 00000000..38d89bc8 --- /dev/null +++ b/brailleblaster-core/src/main/dist/programData/pandoc/lua/docx-page-break.lua @@ -0,0 +1,136 @@ +-- Turn DOCX page breaks into page-break Divs that bbx.lua writes as +-- . +-- +-- Pandoc drops Word page breaks, so DocxPageBreaks replaces each one with +-- MARKER text before pandoc reads the file. This filter splits paragraphs at +-- the marker. A marker it cannot split on (inside formatting, a list or a +-- table) puts the break before the block that holds it. + +local MARKER = "\u{E000}BBPAGEBREAK\u{E000}" +local BREAK_STYLE = "BB Page Break" + +local function pageBreak() + return pandoc.Div({}, pandoc.Attr("", {}, { ["custom-style"] = BREAK_STYLE })) +end + +local function isPageBreak(block) + return block.t == "Div" and block.attr.attributes["custom-style"] == BREAK_STYLE +end + +-- Split a Str into its text pieces and markers. +local function splitStr(text) + local pieces = {} + local start = 1 + while true do + local i, j = string.find(text, MARKER, start, true) + if i == nil then break end + if i > start then pieces[#pieces + 1] = pandoc.Str(string.sub(text, start, i - 1)) end + pieces[#pieces + 1] = "break" + start = j + 1 + end + if start <= #text then pieces[#pieces + 1] = pandoc.Str(string.sub(text, start)) end + return pieces +end + +-- Remove markers anywhere inside a block. Returns the block and the number +-- of markers removed. +local function stripMarkers(block) + local count = 0 + local stripped = pandoc.walk_block(block, { + Str = function(s) + local _, n = string.gsub(s.text, MARKER, "") + if n == 0 then return nil end + count = count + n + local text = string.gsub(s.text, MARKER, "") + return pandoc.Str(text) + end + }) + return stripped, count +end + +local function addBreaks(out, count) + for _ = 1, count do out[#out + 1] = pageBreak() end +end + +local function isBlank(inlines) + for _, inline in ipairs(inlines) do + if inline.t ~= "Space" and inline.t ~= "SoftBreak" and inline.t ~= "LineBreak" + and not (inline.t == "Str" and inline.text == "") then + return false + end + end + return true +end + +-- Split a Para or Plain at top-level markers. +local function splitParagraph(block, out) + local current = {} + local function flush() + if not isBlank(current) then out[#out + 1] = block.t == "Para" and pandoc.Para(current) or pandoc.Plain(current) end + current = {} + end + + for _, inline in ipairs(block.content) do + if inline.t == "Str" and string.find(inline.text, MARKER, 1, true) then + for _, piece in ipairs(splitStr(inline.text)) do + if piece == "break" then + flush() + out[#out + 1] = pageBreak() + else + current[#current + 1] = piece + end + end + else + local stripped, count = stripMarkers(pandoc.Plain({ inline })) + if count > 0 then + -- the break is inside formatting, so it goes before this text + flush() + addBreaks(out, count) + end + for _, x in ipairs(stripped.content) do current[#current + 1] = x end + end + end + flush() +end + +local processBlocks + +-- Split a Div around the breaks in its content, so the breaks stay +-- outside custom-style Divs such as Word's List styles. +local function splitDiv(block, out) + local segment = {} + local function flush() + if #segment > 0 then out[#out + 1] = pandoc.Div(segment, block.attr) end + segment = {} + end + + for _, child in ipairs(processBlocks(block.content)) do + if isPageBreak(child) then + flush() + out[#out + 1] = child + else + segment[#segment + 1] = child + end + end + flush() +end + +processBlocks = function(blocks) + local out = {} + for _, block in ipairs(blocks) do + if block.t == "Para" or block.t == "Plain" then + splitParagraph(block, out) + elseif block.t == "Div" then + splitDiv(block, out) + else + local stripped, count = stripMarkers(block) + addBreaks(out, count) + out[#out + 1] = stripped + end + end + return out +end + +function Pandoc(doc) + return pandoc.Pandoc(processBlocks(doc.blocks), doc.meta) +end diff --git a/brailleblaster-core/src/main/java/org/brailleblaster/archiver2/PandocArchiverLoader.kt b/brailleblaster-core/src/main/java/org/brailleblaster/archiver2/PandocArchiverLoader.kt index f9fa4e1e..a41386de 100644 --- a/brailleblaster-core/src/main/java/org/brailleblaster/archiver2/PandocArchiverLoader.kt +++ b/brailleblaster-core/src/main/java/org/brailleblaster/archiver2/PandocArchiverLoader.kt @@ -16,9 +16,11 @@ package org.brailleblaster.archiver2 import org.brailleblaster.BBIni +import org.brailleblaster.pandoc.DocxPageBreaks import org.brailleblaster.pandoc.FixImage import org.brailleblaster.pandoc.FixMathML import org.brailleblaster.pandoc.FixNestedList +import org.brailleblaster.pandoc.FixPageBreak import org.brailleblaster.pandoc.Fixer import org.brailleblaster.util.PANDOC_CMD import org.slf4j.LoggerFactory @@ -101,9 +103,12 @@ object PandocArchiverLoader : ArchiverFactory.FileLoader { "--from=$fromFormat" ) - // DOCX custom paragraph style "List Paragraph" arrives as Div(custom-style) - // and must be normalized to a real Pandoc list before bbx.lua is applied. - if (fromFormat?.startsWith("docx") == true) { + val isDocx = fromFormat?.startsWith("docx") == true + if (isDocx) { + // Pandoc drops page breaks; keep them as markers for this filter. + command.add("--lua-filter=docx-page-break.lua") + // DOCX custom paragraph style "List Paragraph" arrives as Div(custom-style) + // and must be normalized to a real Pandoc list before bbx.lua is applied. command.add("--lua-filter=list-paragraph-style.lua") } @@ -111,7 +116,7 @@ object PandocArchiverLoader : ArchiverFactory.FileLoader { listOf( "--to=bbx.lua", "--output=" + bbFile.absolutePath, - filename + if (isDocx) DocxPageBreaks.markPageBreaks(File(filename)).absolutePath else filename ) ) @@ -146,6 +151,7 @@ object PandocArchiverLoader : ArchiverFactory.FileLoader { val fixImage = FixImage() val fixMath = FixMathML() val fixer = Fixer(newFilename) + fixer.addFixer(FixPageBreak()) fixer.addFixer(fixMath) fixer.addFixer(fixNestedList) fixer.addFixer(fixImage) diff --git a/brailleblaster-core/src/main/java/org/brailleblaster/pandoc/DocxPageBreaks.kt b/brailleblaster-core/src/main/java/org/brailleblaster/pandoc/DocxPageBreaks.kt new file mode 100644 index 00000000..4ebe9921 --- /dev/null +++ b/brailleblaster-core/src/main/java/org/brailleblaster/pandoc/DocxPageBreaks.kt @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2026 American Printing House for the Blind + * + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program. If not, see . + */ +package org.brailleblaster.pandoc + +import java.io.File +import java.util.zip.ZipEntry +import java.util.zip.ZipFile +import java.util.zip.ZipOutputStream + +/** + * Pandoc drops Word page breaks, so before pandoc reads a DOCX each break is + * replaced with marker text. docx-page-break.lua turns the markers into + * page-break blocks and [FixPageBreak] sets them on the next block. + */ +object DocxPageBreaks { + /** Must match MARKER in docx-page-break.lua. */ + private const val MARKER = "BBPAGEBREAK" + private const val DOCUMENT_XML = "word/document.xml" + + /** ``, a manual page break inside a run. */ + private val PAGE_BREAK = Regex("""]*\bw:type="page"[^>]*/>""") + + /** A paragraph whose own properties turn on "Page break before". */ + private val PAGE_BREAK_BEFORE = Regex( + """(]*>\s*(?:(?!).)*?]*w:val="(?:0|false|off)")[^>]*/>.*?)""", + RegexOption.DOT_MATCHES_ALL + ) + + fun markPageBreaks(xml: String): String = xml + .replace(PAGE_BREAK, "$MARKER") + .replace(PAGE_BREAK_BEFORE) { "${it.groupValues[1]}$MARKER" } + + /** + * Returns a temporary copy of [docx] with its page breaks marked, or [docx] + * itself when it has none. + */ + fun markPageBreaks(docx: File): File = ZipFile(docx).use { zip -> + val documentEntry = zip.getEntry(DOCUMENT_XML) ?: return docx + val xml = zip.getInputStream(documentEntry).use { it.readBytes().toString(Charsets.UTF_8) } + val marked = markPageBreaks(xml) + if (marked == xml) return docx + + File.createTempFile("bb-${docx.nameWithoutExtension}-", ".docx").apply { + deleteOnExit() + ZipOutputStream(outputStream()).use { out -> + zip.entries().asSequence().forEach { entry -> + out.putNextEntry(ZipEntry(entry.name)) + if (entry.name == DOCUMENT_XML) { + out.write(marked.toByteArray(Charsets.UTF_8)) + } else { + zip.getInputStream(entry).use { it.copyTo(out) } + } + out.closeEntry() + } + } + } + } +} diff --git a/brailleblaster-core/src/main/java/org/brailleblaster/pandoc/FixPageBreak.kt b/brailleblaster-core/src/main/java/org/brailleblaster/pandoc/FixPageBreak.kt new file mode 100644 index 00000000..59b7e366 --- /dev/null +++ b/brailleblaster-core/src/main/java/org/brailleblaster/pandoc/FixPageBreak.kt @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2026 American Printing House for the Blind + * + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program. If not, see . + */ +package org.brailleblaster.pandoc + +import nu.xom.Element +import org.brailleblaster.bbx.BBX +import org.brailleblaster.bbx.utd.BBXDynamicOptionStyleMap +import org.brailleblaster.utd.Style.StyleOption +import org.brailleblaster.utils.xml.BB_NS +import org.brailleblaster.utils.xml.UTD_NS + +/** + * Replaces each `` written by bbx.lua with a new page before + * the block that follows it, the same style option the Page Break tool sets. + * A table gets the option on its container and its UTD table copy. + */ +class FixPageBreak : FixerInf { + private lateinit var rootElem: Element + + override fun setFixer(fixer: Fixer) { + rootElem = fixer.rootElement + } + + override fun process() { + val elements = descendants(rootElem).toList() + elements.withIndex() + .filter { (_, e) -> isPageBreak(e) } + .forEach { (i, pageBreak) -> + elements.drop(i + 1) + .firstOrNull { e -> BBX.BLOCK.isA(e) && e.parent != null } + ?.let(::breakTarget) + ?.forEach(::addPageBefore) + pageBreak.detach() + } + } + + private fun descendants(e: Element): Sequence = + sequenceOf(e) + e.childElements.asSequence().flatMap(::descendants) + + private fun isPageBreak(e: Element) = e.localName == "pageBreak" && e.namespaceURI == BB_NS + + private fun breakTarget(block: Element): List { + if (!BBX.BLOCK.TABLE_CELL.isA(block)) return listOf(block) + val table = generateSequence(block.parent as? Element) { it.parent as? Element } + .lastOrNull { BBX.CONTAINER.TABLE.isA(it) } ?: return listOf(block) + val parent = table.parent + val copy = parent.indexOf(table).let { i -> if (i + 1 < parent.childCount) parent.getChild(i + 1) else null } + return listOfNotNull( + table, + (copy as? Element)?.takeIf { it.getAttribute("tableCopy", UTD_NS) != null } + ) + } + + private fun addPageBefore(e: Element) { + val pages = e.getAttributeValue(PAGES_BEFORE_ATTRIB, BB_NS)?.toIntOrNull() ?: 0 + BBXDynamicOptionStyleMap.setStyleOptionAttrib(e, StyleOption.NEW_PAGES_BEFORE, pages + 1) + } + + companion object { + private val PAGES_BEFORE_ATTRIB = + BBXDynamicOptionStyleMap.OPTION_ATTRIB_PREFIX + StyleOption.NEW_PAGES_BEFORE.optionName + } +} diff --git a/brailleblaster-core/src/test/java/org/brailleblaster/pandoc/PandocDocxPageBreakTest.kt b/brailleblaster-core/src/test/java/org/brailleblaster/pandoc/PandocDocxPageBreakTest.kt new file mode 100644 index 00000000..7ea0a611 --- /dev/null +++ b/brailleblaster-core/src/test/java/org/brailleblaster/pandoc/PandocDocxPageBreakTest.kt @@ -0,0 +1,209 @@ +/* + * Copyright (C) 2026 American Printing House for the Blind + * + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program. If not, see . + */ +package org.brailleblaster.pandoc + +import nu.xom.Document +import nu.xom.Element +import org.brailleblaster.BBIni.programDataPath +import org.brailleblaster.bbx.BBX +import org.brailleblaster.bbx.BookToBBXConverter +import org.brailleblaster.util.PANDOC_CMD +import org.brailleblaster.utils.xml.BB_NS +import org.testng.Assert +import org.testng.annotations.BeforeClass +import org.testng.annotations.Test +import java.io.File +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +/** + * Word page breaks must import as a new page before the block that follows + * the break, the same style option the Page Break tool sets. + */ +class PandocDocxPageBreakTest { + private val luaDir: String by lazy { programDataPath.resolve("pandoc").resolve("lua").toString() } + + @BeforeClass + fun init() { + BookToBBXConverter.devSetup(arrayOf()) + } + + /** Mirrors the PandocArchiverLoader command line and fixers for DOCX. */ + private fun runPandoc(input: File): Document { + val bbxFile = File.createTempFile("bb-pagebreak-out-", ".bbx").apply { deleteOnExit() } + val command = listOf( + PANDOC_CMD, + "--from=docx+styles+empty_paragraphs", + "--lua-filter=docx-page-break.lua", + "--lua-filter=list-paragraph-style.lua", + "--to=bbx.lua", + "--output=" + bbxFile.absolutePath, + DocxPageBreaks.markPageBreaks(input).absolutePath + ) + val proc = ProcessBuilder(command) + .directory(File(luaDir)) + .redirectErrorStream(true) + .apply { environment()["PANDOCCMD"] = PANDOC_CMD } + .start() + val output = proc.inputStream.bufferedReader().readText() + Assert.assertEquals(proc.waitFor(), 0, "pandoc failed: $output") + + return Fixer(bbxFile.absolutePath).apply { + addFixer(FixPageBreak()) + addFixer(FixMathML()) + addFixer(FixNestedList()) + addFixer(FixImage()) + processFixers() + }.document + } + + private fun resourceFile(name: String): File = + File.createTempFile("bb-pagebreak-in-", ".docx").apply { + deleteOnExit() + val bytes = PandocDocxPageBreakTest::class.java.getResourceAsStream(name) + ?.use { it.readAllBytes() } + Assert.assertNotNull(bytes, "Test resource $name not found on classpath") + writeBytes(bytes!!) + } + + /** A minimal DOCX whose body is [body]. */ + private fun docx(body: String): File = + File.createTempFile("bb-pagebreak-gen-", ".docx").apply { + deleteOnExit() + val parts = mapOf( + "[Content_Types].xml" to """ + + + + + """, + "_rels/.rels" to """ + + + """, + "word/document.xml" to """ + + $body""" + ) + ZipOutputStream(outputStream()).use { zip -> + parts.forEach { (name, xml) -> + zip.putNextEntry(ZipEntry(name)) + zip.write(xml.trimIndent().toByteArray(Charsets.UTF_8)) + zip.closeEntry() + } + } + } + + private fun para(text: String) = "$text" + + private val pageBreakPara = """""" + + private fun blocks(doc: Document): List = + doc.query("//*[local-name()='BLOCK']").map { it as Element } + + private fun blockWithText(doc: Document, text: String): Element = + blocks(doc).firstOrNull { it.value.trim() == text } + ?: throw AssertionError("No BLOCK with text '$text' in:\n${doc.toXML()}") + + private fun pagesBefore(e: Element): Int? = + e.getAttributeValue("overrideOption-newPagesBefore", BB_NS)?.toInt() + + private fun assertPagesBefore(doc: Document, text: String, pages: Int?) { + val block = blockWithText(doc, text) + Assert.assertEquals(pagesBefore(block), pages, "New pages before '$text':\n${block.toXML()}") + } + + private fun assertNoMarkersLeft(doc: Document) { + Assert.assertEquals(doc.query("//*[local-name()='pageBreak']").size(), 0, "pageBreak left:\n${doc.toXML()}") + Assert.assertFalse(doc.toXML().contains("BBPAGEBREAK"), "Marker text left:\n${doc.toXML()}") + } + + @Test + fun statementPageBreakStartsNewPage() { + val doc = runPandoc(resourceFile("list-style-statement.docx")) + + assertPagesBefore(doc, "Account Activity", 1) + assertPagesBefore(doc, "Annual Percentage Yield Earned 3.123%", null) + Assert.assertEquals( + doc.query("//*[@*[local-name()='overrideOption-newPagesBefore']]").size(), 1, + "The statement has a single page break:\n${doc.toXML()}" + ) + assertNoMarkersLeft(doc) + } + + @Test + fun breakInsideParagraphSplitsIt() { + val doc = runPandoc( + docx("""BeforeAfter""") + ) + + assertPagesBefore(doc, "Before", null) + assertPagesBefore(doc, "After", 1) + assertNoMarkersLeft(doc) + } + + @Test + fun consecutiveBreaksAddPages() { + val doc = runPandoc(docx(para("One") + pageBreakPara + pageBreakPara + para("Two"))) + + assertPagesBefore(doc, "One", null) + assertPagesBefore(doc, "Two", 2) + assertNoMarkersLeft(doc) + } + + @Test + fun pageBreakBeforeParagraphProperty() { + val doc = runPandoc( + docx( + para("One") + + "Two" + + """Three""" + ) + ) + + assertPagesBefore(doc, "One", null) + assertPagesBefore(doc, "Two", 1) + assertPagesBefore(doc, "Three", null) + assertNoMarkersLeft(doc) + } + + @Test + fun breakBeforeTableMarksTableAndCopy() { + val cell = "Cell" + val doc = runPandoc(docx(para("One") + pageBreakPara + "$cell$cell")) + + val tables = doc.query("//*[local-name()='CONTAINER' and @*[local-name()='type']='TABLE']") + .map { it as Element } + Assert.assertEquals(tables.size, 2, "Expected a table and its UTD copy:\n${doc.toXML()}") + tables.forEach { Assert.assertEquals(pagesBefore(it), 1, "Table should start a new page:\n${it.toXML()}") } + Assert.assertTrue(blocks(doc).filter { BBX.BLOCK.TABLE_CELL.isA(it) }.none { pagesBefore(it) != null }) + assertNoMarkersLeft(doc) + } + + @Test + fun breakAtEndIsDropped() { + val doc = runPandoc(docx(para("Only") + pageBreakPara)) + + assertPagesBefore(doc, "Only", null) + assertNoMarkersLeft(doc) + } + + @Test + fun docxWithoutBreaksIsNotCopied() { + val input = docx(para("Plain")) + Assert.assertSame(DocxPageBreaks.markPageBreaks(input), input) + } +}