diff --git a/brailleblaster-core/src/main/dist/programData/pandoc/lua/bbx.lua b/brailleblaster-core/src/main/dist/programData/pandoc/lua/bbx.lua index f8a3d3da..eca45096 100644 --- a/brailleblaster-core/src/main/dist/programData/pandoc/lua/bbx.lua +++ b/brailleblaster-core/src/main/dist/programData/pandoc/lua/bbx.lua @@ -724,6 +724,10 @@ end function Div(s, attr) local customStyle = getCustomStyle(attr) + -- From docx-page-break.lua; FixPageBreak moves it onto the next block. + if customStyle == "BB Page Break" then + return '' + end if isListParagraphStyle(customStyle) or isPlainListStyle(customStyle) then local item = listItemText(s) if string.len(string.gsub(item, '%s+', '')) == 0 then diff --git a/brailleblaster-core/src/main/dist/programData/pandoc/lua/docx-page-break.lua b/brailleblaster-core/src/main/dist/programData/pandoc/lua/docx-page-break.lua new file mode 100644 index 00000000..38d89bc8 --- /dev/null +++ b/brailleblaster-core/src/main/dist/programData/pandoc/lua/docx-page-break.lua @@ -0,0 +1,136 @@ +-- Turn DOCX page breaks into page-break Divs that bbx.lua writes as +-- . +-- +-- Pandoc drops Word page breaks, so DocxPageBreaks replaces each one with +-- MARKER text before pandoc reads the file. This filter splits paragraphs at +-- the marker. A marker it cannot split on (inside formatting, a list or a +-- table) puts the break before the block that holds it. + +local MARKER = "\u{E000}BBPAGEBREAK\u{E000}" +local BREAK_STYLE = "BB Page Break" + +local function pageBreak() + return pandoc.Div({}, pandoc.Attr("", {}, { ["custom-style"] = BREAK_STYLE })) +end + +local function isPageBreak(block) + return block.t == "Div" and block.attr.attributes["custom-style"] == BREAK_STYLE +end + +-- Split a Str into its text pieces and markers. +local function splitStr(text) + local pieces = {} + local start = 1 + while true do + local i, j = string.find(text, MARKER, start, true) + if i == nil then break end + if i > start then pieces[#pieces + 1] = pandoc.Str(string.sub(text, start, i - 1)) end + pieces[#pieces + 1] = "break" + start = j + 1 + end + if start <= #text then pieces[#pieces + 1] = pandoc.Str(string.sub(text, start)) end + return pieces +end + +-- Remove markers anywhere inside a block. Returns the block and the number +-- of markers removed. +local function stripMarkers(block) + local count = 0 + local stripped = pandoc.walk_block(block, { + Str = function(s) + local _, n = string.gsub(s.text, MARKER, "") + if n == 0 then return nil end + count = count + n + local text = string.gsub(s.text, MARKER, "") + return pandoc.Str(text) + end + }) + return stripped, count +end + +local function addBreaks(out, count) + for _ = 1, count do out[#out + 1] = pageBreak() end +end + +local function isBlank(inlines) + for _, inline in ipairs(inlines) do + if inline.t ~= "Space" and inline.t ~= "SoftBreak" and inline.t ~= "LineBreak" + and not (inline.t == "Str" and inline.text == "") then + return false + end + end + return true +end + +-- Split a Para or Plain at top-level markers. +local function splitParagraph(block, out) + local current = {} + local function flush() + if not isBlank(current) then out[#out + 1] = block.t == "Para" and pandoc.Para(current) or pandoc.Plain(current) end + current = {} + end + + for _, inline in ipairs(block.content) do + if inline.t == "Str" and string.find(inline.text, MARKER, 1, true) then + for _, piece in ipairs(splitStr(inline.text)) do + if piece == "break" then + flush() + out[#out + 1] = pageBreak() + else + current[#current + 1] = piece + end + end + else + local stripped, count = stripMarkers(pandoc.Plain({ inline })) + if count > 0 then + -- the break is inside formatting, so it goes before this text + flush() + addBreaks(out, count) + end + for _, x in ipairs(stripped.content) do current[#current + 1] = x end + end + end + flush() +end + +local processBlocks + +-- Split a Div around the breaks in its content, so the breaks stay +-- outside custom-style Divs such as Word's List styles. +local function splitDiv(block, out) + local segment = {} + local function flush() + if #segment > 0 then out[#out + 1] = pandoc.Div(segment, block.attr) end + segment = {} + end + + for _, child in ipairs(processBlocks(block.content)) do + if isPageBreak(child) then + flush() + out[#out + 1] = child + else + segment[#segment + 1] = child + end + end + flush() +end + +processBlocks = function(blocks) + local out = {} + for _, block in ipairs(blocks) do + if block.t == "Para" or block.t == "Plain" then + splitParagraph(block, out) + elseif block.t == "Div" then + splitDiv(block, out) + else + local stripped, count = stripMarkers(block) + addBreaks(out, count) + out[#out + 1] = stripped + end + end + return out +end + +function Pandoc(doc) + return pandoc.Pandoc(processBlocks(doc.blocks), doc.meta) +end diff --git a/brailleblaster-core/src/main/java/org/brailleblaster/archiver2/PandocArchiverLoader.kt b/brailleblaster-core/src/main/java/org/brailleblaster/archiver2/PandocArchiverLoader.kt index f9fa4e1e..a41386de 100644 --- a/brailleblaster-core/src/main/java/org/brailleblaster/archiver2/PandocArchiverLoader.kt +++ b/brailleblaster-core/src/main/java/org/brailleblaster/archiver2/PandocArchiverLoader.kt @@ -16,9 +16,11 @@ package org.brailleblaster.archiver2 import org.brailleblaster.BBIni +import org.brailleblaster.pandoc.DocxPageBreaks import org.brailleblaster.pandoc.FixImage import org.brailleblaster.pandoc.FixMathML import org.brailleblaster.pandoc.FixNestedList +import org.brailleblaster.pandoc.FixPageBreak import org.brailleblaster.pandoc.Fixer import org.brailleblaster.util.PANDOC_CMD import org.slf4j.LoggerFactory @@ -101,9 +103,12 @@ object PandocArchiverLoader : ArchiverFactory.FileLoader { "--from=$fromFormat" ) - // DOCX custom paragraph style "List Paragraph" arrives as Div(custom-style) - // and must be normalized to a real Pandoc list before bbx.lua is applied. - if (fromFormat?.startsWith("docx") == true) { + val isDocx = fromFormat?.startsWith("docx") == true + if (isDocx) { + // Pandoc drops page breaks; keep them as markers for this filter. + command.add("--lua-filter=docx-page-break.lua") + // DOCX custom paragraph style "List Paragraph" arrives as Div(custom-style) + // and must be normalized to a real Pandoc list before bbx.lua is applied. command.add("--lua-filter=list-paragraph-style.lua") } @@ -111,7 +116,7 @@ object PandocArchiverLoader : ArchiverFactory.FileLoader { listOf( "--to=bbx.lua", "--output=" + bbFile.absolutePath, - filename + if (isDocx) DocxPageBreaks.markPageBreaks(File(filename)).absolutePath else filename ) ) @@ -146,6 +151,7 @@ object PandocArchiverLoader : ArchiverFactory.FileLoader { val fixImage = FixImage() val fixMath = FixMathML() val fixer = Fixer(newFilename) + fixer.addFixer(FixPageBreak()) fixer.addFixer(fixMath) fixer.addFixer(fixNestedList) fixer.addFixer(fixImage) diff --git a/brailleblaster-core/src/main/java/org/brailleblaster/pandoc/DocxPageBreaks.kt b/brailleblaster-core/src/main/java/org/brailleblaster/pandoc/DocxPageBreaks.kt new file mode 100644 index 00000000..4ebe9921 --- /dev/null +++ b/brailleblaster-core/src/main/java/org/brailleblaster/pandoc/DocxPageBreaks.kt @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2026 American Printing House for the Blind + * + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program. If not, see . + */ +package org.brailleblaster.pandoc + +import java.io.File +import java.util.zip.ZipEntry +import java.util.zip.ZipFile +import java.util.zip.ZipOutputStream + +/** + * Pandoc drops Word page breaks, so before pandoc reads a DOCX each break is + * replaced with marker text. docx-page-break.lua turns the markers into + * page-break blocks and [FixPageBreak] sets them on the next block. + */ +object DocxPageBreaks { + /** Must match MARKER in docx-page-break.lua. */ + private const val MARKER = "BBPAGEBREAK" + private const val DOCUMENT_XML = "word/document.xml" + + /** ``, a manual page break inside a run. */ + private val PAGE_BREAK = Regex("""]*\bw:type="page"[^>]*/>""") + + /** A paragraph whose own properties turn on "Page break before". */ + private val PAGE_BREAK_BEFORE = Regex( + """(]*>\s*(?:(?!).)*?]*w:val="(?:0|false|off)")[^>]*/>.*?)""", + RegexOption.DOT_MATCHES_ALL + ) + + fun markPageBreaks(xml: String): String = xml + .replace(PAGE_BREAK, "$MARKER") + .replace(PAGE_BREAK_BEFORE) { "${it.groupValues[1]}$MARKER" } + + /** + * Returns a temporary copy of [docx] with its page breaks marked, or [docx] + * itself when it has none. + */ + fun markPageBreaks(docx: File): File = ZipFile(docx).use { zip -> + val documentEntry = zip.getEntry(DOCUMENT_XML) ?: return docx + val xml = zip.getInputStream(documentEntry).use { it.readBytes().toString(Charsets.UTF_8) } + val marked = markPageBreaks(xml) + if (marked == xml) return docx + + File.createTempFile("bb-${docx.nameWithoutExtension}-", ".docx").apply { + deleteOnExit() + ZipOutputStream(outputStream()).use { out -> + zip.entries().asSequence().forEach { entry -> + out.putNextEntry(ZipEntry(entry.name)) + if (entry.name == DOCUMENT_XML) { + out.write(marked.toByteArray(Charsets.UTF_8)) + } else { + zip.getInputStream(entry).use { it.copyTo(out) } + } + out.closeEntry() + } + } + } + } +} diff --git a/brailleblaster-core/src/main/java/org/brailleblaster/pandoc/FixPageBreak.kt b/brailleblaster-core/src/main/java/org/brailleblaster/pandoc/FixPageBreak.kt new file mode 100644 index 00000000..59b7e366 --- /dev/null +++ b/brailleblaster-core/src/main/java/org/brailleblaster/pandoc/FixPageBreak.kt @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2026 American Printing House for the Blind + * + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program. If not, see . + */ +package org.brailleblaster.pandoc + +import nu.xom.Element +import org.brailleblaster.bbx.BBX +import org.brailleblaster.bbx.utd.BBXDynamicOptionStyleMap +import org.brailleblaster.utd.Style.StyleOption +import org.brailleblaster.utils.xml.BB_NS +import org.brailleblaster.utils.xml.UTD_NS + +/** + * Replaces each `` written by bbx.lua with a new page before + * the block that follows it, the same style option the Page Break tool sets. + * A table gets the option on its container and its UTD table copy. + */ +class FixPageBreak : FixerInf { + private lateinit var rootElem: Element + + override fun setFixer(fixer: Fixer) { + rootElem = fixer.rootElement + } + + override fun process() { + val elements = descendants(rootElem).toList() + elements.withIndex() + .filter { (_, e) -> isPageBreak(e) } + .forEach { (i, pageBreak) -> + elements.drop(i + 1) + .firstOrNull { e -> BBX.BLOCK.isA(e) && e.parent != null } + ?.let(::breakTarget) + ?.forEach(::addPageBefore) + pageBreak.detach() + } + } + + private fun descendants(e: Element): Sequence = + sequenceOf(e) + e.childElements.asSequence().flatMap(::descendants) + + private fun isPageBreak(e: Element) = e.localName == "pageBreak" && e.namespaceURI == BB_NS + + private fun breakTarget(block: Element): List { + if (!BBX.BLOCK.TABLE_CELL.isA(block)) return listOf(block) + val table = generateSequence(block.parent as? Element) { it.parent as? Element } + .lastOrNull { BBX.CONTAINER.TABLE.isA(it) } ?: return listOf(block) + val parent = table.parent + val copy = parent.indexOf(table).let { i -> if (i + 1 < parent.childCount) parent.getChild(i + 1) else null } + return listOfNotNull( + table, + (copy as? Element)?.takeIf { it.getAttribute("tableCopy", UTD_NS) != null } + ) + } + + private fun addPageBefore(e: Element) { + val pages = e.getAttributeValue(PAGES_BEFORE_ATTRIB, BB_NS)?.toIntOrNull() ?: 0 + BBXDynamicOptionStyleMap.setStyleOptionAttrib(e, StyleOption.NEW_PAGES_BEFORE, pages + 1) + } + + companion object { + private val PAGES_BEFORE_ATTRIB = + BBXDynamicOptionStyleMap.OPTION_ATTRIB_PREFIX + StyleOption.NEW_PAGES_BEFORE.optionName + } +} diff --git a/brailleblaster-core/src/test/java/org/brailleblaster/pandoc/PandocDocxPageBreakTest.kt b/brailleblaster-core/src/test/java/org/brailleblaster/pandoc/PandocDocxPageBreakTest.kt new file mode 100644 index 00000000..7ea0a611 --- /dev/null +++ b/brailleblaster-core/src/test/java/org/brailleblaster/pandoc/PandocDocxPageBreakTest.kt @@ -0,0 +1,209 @@ +/* + * Copyright (C) 2026 American Printing House for the Blind + * + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program. If not, see . + */ +package org.brailleblaster.pandoc + +import nu.xom.Document +import nu.xom.Element +import org.brailleblaster.BBIni.programDataPath +import org.brailleblaster.bbx.BBX +import org.brailleblaster.bbx.BookToBBXConverter +import org.brailleblaster.util.PANDOC_CMD +import org.brailleblaster.utils.xml.BB_NS +import org.testng.Assert +import org.testng.annotations.BeforeClass +import org.testng.annotations.Test +import java.io.File +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +/** + * Word page breaks must import as a new page before the block that follows + * the break, the same style option the Page Break tool sets. + */ +class PandocDocxPageBreakTest { + private val luaDir: String by lazy { programDataPath.resolve("pandoc").resolve("lua").toString() } + + @BeforeClass + fun init() { + BookToBBXConverter.devSetup(arrayOf()) + } + + /** Mirrors the PandocArchiverLoader command line and fixers for DOCX. */ + private fun runPandoc(input: File): Document { + val bbxFile = File.createTempFile("bb-pagebreak-out-", ".bbx").apply { deleteOnExit() } + val command = listOf( + PANDOC_CMD, + "--from=docx+styles+empty_paragraphs", + "--lua-filter=docx-page-break.lua", + "--lua-filter=list-paragraph-style.lua", + "--to=bbx.lua", + "--output=" + bbxFile.absolutePath, + DocxPageBreaks.markPageBreaks(input).absolutePath + ) + val proc = ProcessBuilder(command) + .directory(File(luaDir)) + .redirectErrorStream(true) + .apply { environment()["PANDOCCMD"] = PANDOC_CMD } + .start() + val output = proc.inputStream.bufferedReader().readText() + Assert.assertEquals(proc.waitFor(), 0, "pandoc failed: $output") + + return Fixer(bbxFile.absolutePath).apply { + addFixer(FixPageBreak()) + addFixer(FixMathML()) + addFixer(FixNestedList()) + addFixer(FixImage()) + processFixers() + }.document + } + + private fun resourceFile(name: String): File = + File.createTempFile("bb-pagebreak-in-", ".docx").apply { + deleteOnExit() + val bytes = PandocDocxPageBreakTest::class.java.getResourceAsStream(name) + ?.use { it.readAllBytes() } + Assert.assertNotNull(bytes, "Test resource $name not found on classpath") + writeBytes(bytes!!) + } + + /** A minimal DOCX whose body is [body]. */ + private fun docx(body: String): File = + File.createTempFile("bb-pagebreak-gen-", ".docx").apply { + deleteOnExit() + val parts = mapOf( + "[Content_Types].xml" to """ + + + + + """, + "_rels/.rels" to """ + + + """, + "word/document.xml" to """ + + $body""" + ) + ZipOutputStream(outputStream()).use { zip -> + parts.forEach { (name, xml) -> + zip.putNextEntry(ZipEntry(name)) + zip.write(xml.trimIndent().toByteArray(Charsets.UTF_8)) + zip.closeEntry() + } + } + } + + private fun para(text: String) = "$text" + + private val pageBreakPara = """""" + + private fun blocks(doc: Document): List = + doc.query("//*[local-name()='BLOCK']").map { it as Element } + + private fun blockWithText(doc: Document, text: String): Element = + blocks(doc).firstOrNull { it.value.trim() == text } + ?: throw AssertionError("No BLOCK with text '$text' in:\n${doc.toXML()}") + + private fun pagesBefore(e: Element): Int? = + e.getAttributeValue("overrideOption-newPagesBefore", BB_NS)?.toInt() + + private fun assertPagesBefore(doc: Document, text: String, pages: Int?) { + val block = blockWithText(doc, text) + Assert.assertEquals(pagesBefore(block), pages, "New pages before '$text':\n${block.toXML()}") + } + + private fun assertNoMarkersLeft(doc: Document) { + Assert.assertEquals(doc.query("//*[local-name()='pageBreak']").size(), 0, "pageBreak left:\n${doc.toXML()}") + Assert.assertFalse(doc.toXML().contains("BBPAGEBREAK"), "Marker text left:\n${doc.toXML()}") + } + + @Test + fun statementPageBreakStartsNewPage() { + val doc = runPandoc(resourceFile("list-style-statement.docx")) + + assertPagesBefore(doc, "Account Activity", 1) + assertPagesBefore(doc, "Annual Percentage Yield Earned 3.123%", null) + Assert.assertEquals( + doc.query("//*[@*[local-name()='overrideOption-newPagesBefore']]").size(), 1, + "The statement has a single page break:\n${doc.toXML()}" + ) + assertNoMarkersLeft(doc) + } + + @Test + fun breakInsideParagraphSplitsIt() { + val doc = runPandoc( + docx("""BeforeAfter""") + ) + + assertPagesBefore(doc, "Before", null) + assertPagesBefore(doc, "After", 1) + assertNoMarkersLeft(doc) + } + + @Test + fun consecutiveBreaksAddPages() { + val doc = runPandoc(docx(para("One") + pageBreakPara + pageBreakPara + para("Two"))) + + assertPagesBefore(doc, "One", null) + assertPagesBefore(doc, "Two", 2) + assertNoMarkersLeft(doc) + } + + @Test + fun pageBreakBeforeParagraphProperty() { + val doc = runPandoc( + docx( + para("One") + + "Two" + + """Three""" + ) + ) + + assertPagesBefore(doc, "One", null) + assertPagesBefore(doc, "Two", 1) + assertPagesBefore(doc, "Three", null) + assertNoMarkersLeft(doc) + } + + @Test + fun breakBeforeTableMarksTableAndCopy() { + val cell = "Cell" + val doc = runPandoc(docx(para("One") + pageBreakPara + "$cell$cell")) + + val tables = doc.query("//*[local-name()='CONTAINER' and @*[local-name()='type']='TABLE']") + .map { it as Element } + Assert.assertEquals(tables.size, 2, "Expected a table and its UTD copy:\n${doc.toXML()}") + tables.forEach { Assert.assertEquals(pagesBefore(it), 1, "Table should start a new page:\n${it.toXML()}") } + Assert.assertTrue(blocks(doc).filter { BBX.BLOCK.TABLE_CELL.isA(it) }.none { pagesBefore(it) != null }) + assertNoMarkersLeft(doc) + } + + @Test + fun breakAtEndIsDropped() { + val doc = runPandoc(docx(para("Only") + pageBreakPara)) + + assertPagesBefore(doc, "Only", null) + assertNoMarkersLeft(doc) + } + + @Test + fun docxWithoutBreaksIsNotCopied() { + val input = docx(para("Plain")) + Assert.assertSame(DocxPageBreaks.markPageBreaks(input), input) + } +}