diff --git a/.github/workflows/release-all-tools.yml b/.github/workflows/release-all-tools.yml index 1e9c70f54..2825b0a90 100644 --- a/.github/workflows/release-all-tools.yml +++ b/.github/workflows/release-all-tools.yml @@ -3,6 +3,7 @@ name: Combined release tools on: pull_request: paths: + - 'matrix-parquet/**' - 'releaseAll.sh' - 'scripts/release-all/**' - 'matrix-bom/verifyBomApi.sh' @@ -73,3 +74,26 @@ jobs: run: | BOM_VERIFY_REPO="$GITHUB_WORKSPACE/build/releaseAll/bom-verify" bash matrix-bom/verifyBomApi.sh groovy scripts/release-all/test-manifest-verifier.groovy . build/releaseAll/bom-verify + + parquet-interop: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - uses: gradle/actions/setup-gradle@v4 + - run: python -m pip install pyarrow==21.0.0 huggingface_hub==2.2.0 + - run: python matrix-parquet/src/test/python/generate_optimized.py + - run: python matrix-parquet/src/test/python/generate_hub_urls.py + - run: git diff --exit-code -- matrix-parquet/src/test/resources/optimized + - run: ./gradlew :matrix-parquet:codenarcMain :matrix-parquet:codenarcTest :matrix-parquet:spotlessCheck :matrix-parquet:test :matrix-parquet:compileConsumerApi -PrunSlowTests=true + env: + PARQUET_PYTHON: python + - run: python matrix-parquet/src/test/python/verify_matrix_output.py + - run: ./gradlew :matrix-parquet:test --tests ParquetInteropTest --rerun-tasks diff --git a/docs/cookbook/matrix-parquet.md b/docs/cookbook/matrix-parquet.md index a7c6ba5b3..41ffdf6b8 100644 --- a/docs/cookbook/matrix-parquet.md +++ b/docs/cookbook/matrix-parquet.md @@ -158,3 +158,29 @@ For files with no Matrix metadata at all, Parquet `DECIMAL` columns are inferred --- [Back to index](cookbook.md) + + +## Optimized Parquet, projection and bounded reads (0.8.0) + +```groovy +import se.alipsa.matrix.parquet.MatrixParquetReader +import se.alipsa.matrix.parquet.Predicates + +def subset = MatrixParquetReader.builder().columns(['id', 'text']) + .filter(Predicates.ge('id', 1000L)).limit(100) + .read(new File('downloaded-optimized.parquet')) + +MatrixParquetReader.builder().maxBatchRows(500).batches(new File('downloaded-optimized.parquet')) + .withCloseable { batches -> + while (batches.hasNext()) { + def batch = batches.next() + // Process this Matrix before requesting another batch. + } + } +``` + +CDC pages decode as standard Parquet; page indexes help eligible predicates skip pages. These APIs +include exact logical-schema predicate conversion and retain selected types for empty results. +See the [complete module guide](../../matrix-parquet/readme.md) for all parameter defaults, +HTTP/Hub access, sharded datasets, writer tuning, fixture reproduction and the external CDC rewrite. +Read the [0.8.0 migration table](../../matrix-parquet/release.md) before upgrading existing calls. diff --git a/docs/tutorial/11-matrix-parquet.md b/docs/tutorial/11-matrix-parquet.md index a11261085..2850a1e6f 100644 --- a/docs/tutorial/11-matrix-parquet.md +++ b/docs/tutorial/11-matrix-parquet.md @@ -261,3 +261,29 @@ The module uses Apache Parquet and Hadoop libraries directly: In the next section, we'll explore the matrix-bigquery module, which provides functionality for interacting with Google BigQuery. Go to [previous section](10-matrix-bom.md) | Go to [next section](12-matrix-bigquery.md) | Back to [outline](outline.md) + + +## Optimized Parquet, projection and bounded reads (0.8.0) + +```groovy +import se.alipsa.matrix.parquet.MatrixParquetReader +import se.alipsa.matrix.parquet.Predicates + +def subset = MatrixParquetReader.builder().columns(['id', 'text']) + .filter(Predicates.ge('id', 1000L)).limit(100) + .read(new File('downloaded-optimized.parquet')) + +MatrixParquetReader.builder().maxBatchRows(500).batches(new File('downloaded-optimized.parquet')) + .withCloseable { batches -> + while (batches.hasNext()) { + def batch = batches.next() + // Process this Matrix before requesting another batch. + } + } +``` + +CDC pages decode as standard Parquet; page indexes help eligible predicates skip pages. These APIs +include exact logical-schema predicate conversion and retain selected types for empty results. +See the [complete module guide](../../matrix-parquet/readme.md) for all parameter defaults, +HTTP/Hub access, sharded datasets, writer tuning, fixture reproduction and the external CDC rewrite. +Read the [0.8.0 migration table](../../matrix-parquet/release.md) before upgrading existing calls. diff --git a/matrix-bom/bom.xml b/matrix-bom/bom.xml index c34d6cc9a..5202867c5 100644 --- a/matrix-bom/bom.xml +++ b/matrix-bom/bom.xml @@ -46,7 +46,7 @@ 2.3.3-SNAPSHOT 0.1.1-SNAPSHOT 0.1.3-SNAPSHOT - 0.7.1-SNAPSHOT + 0.8.0-SNAPSHOT 0.7.0-SNAPSHOT 2.5.1-SNAPSHOT 2.5.1-SNAPSHOT diff --git a/matrix-parquet/benchmarks/optimized-parquet.md b/matrix-parquet/benchmarks/optimized-parquet.md new file mode 100644 index 000000000..f7591f476 --- /dev/null +++ b/matrix-parquet/benchmarks/optimized-parquet.md @@ -0,0 +1,145 @@ +# Optimized Parquet measurements + +Measured 2026-10-17 on macOS arm64 (Apple M5 Max), G1 GC, OpenJDK 21.0.12.1, 4 GiB test heap, parquet-java 1.18.1, +SNAPPY, default 128 MiB row-group target and 1 MiB pages. Each workload has 1,000,000 rows, +an Integer ID and six String columns. Wide text repeats 1,000 labels per column; categorical +repeats 12. Mixed adds LocalDate and Map(label, exact BigDecimal) columns. Generated inputs +live in build/, outside git. The compared Group loop uses shared corrected value conversion. + +| Workload | File bytes | Group median ms | Columnar median ms | Lower median time | Group retained bytes | Columnar retained bytes | +|---|---:|---:|---:|---:|---:|---:| +| wide | 4,898,180 | 7,990 | 767 | 90.4% | 381,674,424 | 45,751,352 | +| categorical | 4,173,171 | 7,940 | 775 | 90.2% | 342,270,976 | 45,364,064 | +| mixed/nested | 7,934,477 | 18,569 | 2,462 | 86.7% | 790,769,704 | 406,370,144 | + +Medians exclude run 0; three warmed runs are included. Group warmed elapsed ranges were +7,368–10,150 / 6,845–11,093 / 18,334–21,881 ms. Columnar ranges were 765–782 / 763–783 / +2,332–2,486 ms. One codec-spike build overlapped the wide Group run and contributes to its +variance; these are local development measurements, not universal speed guarantees. The +original pre-change reader baseline was separately captured before production edits at +5,633 / 5,413 / 5,433 / 5,424 ms on the same million-row wide shape (file 4,898,111 bytes). + +Current-thread allocated bytes were approximately 13.83 GB versus 1.32 GB (wide), 13.78 GB +versus 1.31 GB (categorical), and 38.53 GB versus 3.52 GB (mixed). Retained deltas are sampled +after explicit full GC with the result live. They measure retained result heap, not peak RSS +or a rigorously sampled peak-live-heap metric. In particular the categorical retained result +uses 86.7% less heap; do not relabel that measurement as peak heap. No OS cache purge was +performed. Run 0 represents first application access, not a guaranteed cold filesystem cache. +Timing thresholds are absent from correctness CI. + +## Confirmation with GC live-heap sampling + +A second million-row run added GC notifications. The sampler records the maximum sum of heap +pool usage **after GC** during each read, including the final full GC with the result live. +This is sampled peak live heap, including benchmark inputs/buffers, not instantaneous heap +occupancy between GCs or process RSS. Medians below exclude run 0 (three warmed runs). + +| Workload | Group ms | Columnar ms | Group sampled live peak bytes | Columnar sampled live peak bytes | +|---|---:|---:|---:|---:| +| wide | 7,280 | 764 | 1,009,640,184 | 584,545,248 | +| categorical | 7,158 | 797 | 963,066,680 | 541,390,488 | +| mixed/nested | 18,936 | 2,238 | 1,998,185,928 | 1,343,803,744 | + +Wide elapsed time is 89.5% lower; categorical sampled live heap is 43.8% lower; mixed elapsed +time is 88.2% lower. Thus the initial 20% elapsed / 25% live-heap / at-most-5% regression targets +are met on these workloads with this stated sampling method. Warmed elapsed ranges were +7,230–7,295 / 6,922–7,263 / 18,605–19,304 ms for Group and 752–786 / 776–800 / 2,213–2,306 ms +for columnar. Allocations were approximately 14.20/1.32, 14.15/1.31 and 40.16/3.40 GB respectively. +These local measurements remain evidence, not performance guarantees or CI timing assertions. + +## Reproduction + +Create an init file outside the repository: + +```groovy +// /tmp/parquet-benchmark.init.gradle +gradle.projectsEvaluated { + gradle.rootProject.project(':matrix-parquet').tasks.named('test') { + systemProperty 'parquet.baseline', 'true' + maxHeapSize = '4g' + } +} +``` + +```bash +./gradlew :matrix-parquet:test --tests '*ParquetReadBenchmarkTest' \ + -I /tmp/parquet-benchmark.init.gradle --no-configuration-cache --console=plain +``` + +The BENCH records include workload, implementation, run, rows, file bytes, elapsed milliseconds, +thread allocation and retained heap. Avoid concurrent builds for isolated timing measurements. + +## Correctness and I/O evidence + +`ParquetPruningTest` inspects actual offset/column indexes and multiple pages, compares indexed +and nonindexed byte counts with row-group/statistics/dictionary/bloom pruning disabled, and +checks dictionary String identity. `ParquetDatasetAndHttpTest` counts transferred range bytes +for narrow projection and selective filtering, checks fallback and ETag changes, and compares +ordered sequential/parallel shard results. These assertions use bytes/results, not wall clocks. + +The no-Hadoop spike (`./gradlew :matrix-parquet:codecWithoutHadoop`) found that SNAPPY, ZSTD, +GZIP and UNCOMPRESSED writer initialization all require +`org/apache/hadoop/io/compress/zlib/ZlibCompressor$CompressionLevel`. Therefore Hadoop remains +on the runtime classpath even though local InputFile/OutputFile use PlainParquetConfiguration. + +The public Java consumer probe (`./gradlew :matrix-parquet:compileConsumerApi`) removes +parquet-java and Hadoop from its compile classpath and successfully compiles projection, +Predicates, limits and writer tuning. This verifies that new public predicates do not require +consumers to add parquet-column explicitly. + +## Bounded dataset parallelism + +Four shards of 250,000 rows each (Integer ID and 1,000 categorical text labels), SNAPPY, +maxBatchRows=65,536, all 1,000,000 rows materialized in deterministic order: + +| Workers | Warm ms (runs 1–3) | Median ms | Median sampled live peak bytes | +|---|---|---:|---:| +| 1 | 370 / 341 / 308 | 341 | 66,905,816 | +| 4 | 247 / 242 / 239 | 242 | 57,235,736 | + +This local workload gains about 29% median throughput time with four workers. Heap samples vary +with collection timing; the measured queue stays bounded to one batch per worker, not a claim +that parallel reads always use less memory. The benchmark verifies first/last IDs and row count. +Run `*ParquetDatasetBenchmarkTest` with the same init setup but system property +`parquet.datasetBenchmark=true` instead of `parquet.baseline`. + +Observed deterministic I/O checks on the controlled fixtures: page-index reads 2,914 bytes +versus 82,700 without page pruning; bloom-filter reads 16,737 versus 40,373 with statistics +unable to exclude the interior key and dictionaries/pages disabled; selective HTTP transferred +88,476 versus a 906,195-byte full object. HTTP data uses generated UUID strings, so exact byte +counts vary; assertions require reduced transfer and exact returned rows, not these constants. + +## Identical Arrow layouts: full scan, projection and predicate + +PyArrow 21.0.0 generated 1,000,000 sorted int64 IDs and six text columns (1,000 repeated labels, +about 140 characters/label), four 250,000-row groups, SNAPPY, 64 KiB target pages, default +dictionary encoding and default CDC sizes. Only CDC/index settings differ. No OS cache purge; +run 0 is warmup, medians include runs 1–3. Heap is the same GC live sample defined above. + +| Layout | Mode | Returned rows | File bytes | Read bytes (footer included) | Median ms | Median live peak bytes | +|---|---|---:|---:|---:|---:|---:| +| ordinary | full | 1,000,000 | 6,066,382 | 6,066,378 | 727 | 110,855,320 | +| ordinary | id projection | 1,000,000 | 6,066,382 | 5,085,954 | 214 | 55,266,648 | +| ordinary | id >= 999900 | 100 | 6,066,382 | 1,279,020 | 99 | 32,019,888 | +| indexed | full | 1,000,000 | 6,069,245 | 6,021,834 | 741 | 117,800,072 | +| indexed | id projection | 1,000,000 | 6,069,245 | 5,082,882 | 226 | 56,010,848 | +| indexed | id >= 999900 | 100 | 6,069,245 | 552,869 | 7 | 27,740,176 | +| CDC + indexed | full | 1,000,000 | 7,250,428 | 6,968,991 | 693 | 111,151,624 | +| CDC + indexed | id projection | 1,000,000 | 7,250,428 | 5,105,311 | 209 | 57,589,672 | +| CDC + indexed | id >= 999900 | 100 | 7,250,428 | 542,383 | 3 | 27,766,232 | + +The selective read benefits from row-group pruning in every layout and page pruning where +indexes exist. These short timings demonstrate this workload, not a guaranteed CDC speedup; +CDC has a larger file here. The independent page-index test isolates page pruning alone. + +Reproduce with pinned PyArrow and the same init script as above, replacing its system property +with `parquet.layoutBenchmark=true`: + +```bash +/tmp/matrix-parquet-fixtures/bin/python matrix-parquet/src/test/python/generate_benchmarks.py +./gradlew :matrix-parquet:test --tests '*ParquetLayoutBenchmarkTest' \ + -I /tmp/parquet-layout.init.gradle --no-configuration-cache --console=plain +``` + +The generator writes only build/layout-benchmarks. The optional test checks returned counts and +last IDs and emits per-run time, exact input bytes and sampled live heap for every combination. diff --git a/matrix-parquet/build.gradle b/matrix-parquet/build.gradle index 177374de0..094566f83 100644 --- a/matrix-parquet/build.gradle +++ b/matrix-parquet/build.gradle @@ -7,7 +7,7 @@ plugins { } group = 'se.alipsa.matrix' -version = '0.7.1-SNAPSHOT' +version = '0.8.0-SNAPSHOT' description = "Groovy library for importing and exporting parquet files to and from a Matrix" JavaCompile javaCompile = compileJava { @@ -124,3 +124,48 @@ nexusReleasePlugin { password = project.ext.properties.sonatypePassword mavenPublication = publishing.publications.maven } + +// Generate the default-size CDC case outside the repository only for the slow lane. +boolean runCdcSlowTests = providers.gradleProperty('runSlowTests').orNull == 'true' +File parquetProjectDirectory = projectDir +tasks.register('generateDefaultCdcFixture') { + onlyIf { runCdcSlowTests } + doLast { + String python = System.getenv('PARQUET_PYTHON') ?: 'python3' + try { + def probe = new ProcessBuilder(python, '-c', 'import pyarrow; assert pyarrow.__version__ == "21.0.0"').inheritIO().start() + if (probe.waitFor() != 0) { + logger.lifecycle('SKIP default CDC fixture: pinned PyArrow 21.0.0 is unavailable') + return + } + } catch (IOException ignored) { + logger.lifecycle('SKIP default CDC fixture: Python is unavailable') + return + } + def generation = new ProcessBuilder(python, 'src/test/python/generate_optimized.py', + 'build/default-cdc', '--default-cdc').directory(parquetProjectDirectory).inheritIO().start() + if (generation.waitFor() != 0) { throw new GradleException('CDC fixture generation failed') } + } +} +tasks.named('test') { dependsOn 'generateDefaultCdcFixture' } + +// Diagnostic spike: report runtime classes still required by upstream codec/writer code. +tasks.register('codecWithoutHadoop', Test) { + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath.filter { !it.name.startsWith('hadoop-') } + useJUnitPlatform() + include '**/ParquetCodecProbeTest.class' + systemProperty 'parquet.codecSpike', 'true' + testLogging.showStandardStreams = true +} + +tasks.register('compileConsumerApi', JavaCompile) { + dependsOn classes + source = fileTree('src/consumerTest/java') { include '**/*.java' } + classpath = sourceSets.test.compileClasspath.filter { + !it.name.startsWith('parquet-') && !it.name.startsWith('hadoop-') + } + sourceSets.main.output + destinationDirectory = layout.buildDirectory.dir('consumer-api') + options.release = 21 +} +tasks.named('check') { dependsOn 'compileConsumerApi' } diff --git a/matrix-parquet/readme.md b/matrix-parquet/readme.md index 9ddfcd525..9d32c5c9c 100644 --- a/matrix-parquet/readme.md +++ b/matrix-parquet/readme.md @@ -1,340 +1,244 @@ -# Matrix-Parquet +# Matrix Parquet -This module enables import and export of [Apache Parquet](https://parquet.apache.org/) files to and from Matrix objects using `MatrixParquetWriter` and `MatrixParquetReader`. +Read and write Apache Parquet from Groovy/Java Matrix applications on JDK 21. This development +version targets **0.8.0** and uses parquet-java 1.18.1. Include `matrix-core` and Groovy alongside +`matrix-parquet` in your application. To test this checkout, run `./gradlew :matrix-parquet:test`. -## At a Glance +## Optimized Parquet and selective reads -| Goal | Entry point | -|------|-------------| -| Read a file | `MatrixParquetReader.builder().read(file)` | -| Write a file | `MatrixParquetWriter.builder(matrix).write(file)` | -| Read via SPI | `Matrix.read(file)` | -| Write via SPI | `matrix.write(file)` | -| Discover read options | `Matrix.listReadOptions('parquet')` / `ParquetReadOptions.describe()` | -| Discover write options | `Matrix.listWriteOptions('parquet')` / `ParquetWriteOptions.describe()` | +Hugging Face ordinary and optimized downloads are standard Parquet. Content-defined chunking +(CDC) changes writer page boundaries for deduplication; it requires no special reader. Page +indexes are independent and let compatible predicates skip pages. The checked-in PyArrow 21 +fixtures exercise all four CDC/index combinations with SNAPPY/ZSTD and V1/V2 pages. -## API Surface - -**Direct API (recommended)** -- `MatrixParquetReader.builder()` — fluent reader with `.matrixName()`, `.zoneId()`, `.read(File/Path/URL/InputStream/byte[])` -- `MatrixParquetWriter.builder(matrix)` — fluent writer with `.precision()`, `.scale()`, `.decimalMeta()`, `.roundingMode()`, `.compressionCodec()`, `.zoneId()`, `.inferPrecisionAndScale()`, `.write(File/Path/OutputStream)`, `.writeBytes()` - -**SPI (generic Matrix API)** -- `Matrix.read(options, file)` — options map with keys `matrixName`, `zoneId` -- `matrix.write(options, file)` — options map with keys `inferPrecisionAndScale`, `precision`, `scale`, `decimalMeta`, `roundingMode`, `compressionCodec`, `zoneId` - -**Typed options classes** -- `ParquetReadOptions` — typed options for reading; use `ParquetReadOptions.describe()` for runtime discovery -- `ParquetWriteOptions` — typed options for writing; use `ParquetWriteOptions.describe()` for runtime discovery - -## Default Behavior - -**Reading** -- Matrix name for file reads: `matrixName` option wins; otherwise the file basename without extension (e.g. `data.parquet` → matrix named `data`) -- Matrix name for stream/byte-array/URL reads: `matrixName` option wins; otherwise the Parquet schema/message name when available -- Timezone: system default; override with `zoneId` option - -**Writing** -- Parquet message type name: `matrix.matrixName` when present, otherwise `MatrixSchema` -- BigDecimal: precision and scale are inferred from the data by default (`inferPrecisionAndScale = true`) -- Explicit DECIMAL scale conversion: `roundingMode = UNNECESSARY` by default, so a discarded non-zero fraction fails instead of being silently rounded -- Compression: `SNAPPY` by default; override with the `compressionCodec` option (e.g. `GZIP`, `ZSTD`, `UNCOMPRESSED`) -- Timezone: system default; override with `zoneId` option -- When the target is an existing directory, the Matrix name is converted to one safe filename inside it. Path separators and unsafe characters are replaced, trailing dots/spaces are removed, blank or dots-only names become `matrix.parquet`, and an existing `.parquet` suffix is preserved. Pass an explicit file (including a non-existent file path) to retain the exact filename. -- Nested Map columns: stored as MAP when value types are homogeneous, as STRUCT when heterogeneous -- Index columns: written to `matrix.indexColumns` metadata as a JSON string array so column names containing commas round-trip; readers also accept the legacy comma-delimited metadata form - -## Installation - -Add the following to your Gradle build script: -```groovy -implementation 'org.apache.groovy:groovy:5.0.6' -implementation 'se.alipsa.matrix:matrix-core:3.9.0' -implementation 'se.alipsa.matrix:matrix-parquet:0.7.0' -``` - -## Basic Usage - -### Using Matrix.read() / matrix.write() - -If `matrix-parquet` is on the classpath, `.parquet` files are also available through the generic Matrix SPI API: - -```groovy -import se.alipsa.matrix.core.Matrix - -Matrix data = Matrix.read(new File('data.parquet')) -Matrix stockholm = Matrix.read([matrixName: 'events', zoneId: 'Europe/Stockholm'], new File('events.parquet')) - -data.write([inferPrecisionAndScale: true], new File('copy.parquet')) -data.write([precision: 38, scale: 18, zoneId: 'Europe/Stockholm'], new File('money.parquet')) -data.write([decimalMeta: [amount: [8, 2]]], new File('money-meta.parquet')) - -println Matrix.listReadOptions('parquet') -println Matrix.listWriteOptions('parquet') -``` - -### Reading a Parquet file ```groovy +import java.time.LocalDate import se.alipsa.matrix.core.Matrix import se.alipsa.matrix.parquet.MatrixParquetReader - -File file = new File("data.parquet") -Matrix matrix = MatrixParquetReader.read(file) - -// Or with a custom matrix name -Matrix matrix = MatrixParquetReader.read(file, "myData") - -// Read from a Path -Matrix matrix = MatrixParquetReader.read(file.toPath()) - -// Read from an InputStream -new FileInputStream(file).withCloseable { is -> - Matrix matrix = MatrixParquetReader.read(is) -} - -// Read from a URL (e.g. file:// or https://) -URL url = file.toURI().toURL() -Matrix matrix = MatrixParquetReader.read(url) - -// Read from a file path string -Matrix matrix = MatrixParquetReader.readFile("/path/to/data.parquet") -``` - -### Writing a Matrix to a Parquet file -```groovy -import se.alipsa.matrix.core.Matrix import se.alipsa.matrix.parquet.MatrixParquetWriter -import se.alipsa.matrix.datasets.Dataset +import se.alipsa.matrix.parquet.Predicates -Matrix data = Dataset.cars().withMatrixName('cars') -File file = new File("cars.parquet") -MatrixParquetWriter.write(data, file) +File file = new File('events.parquet') +Matrix events = Matrix.builder('events').columns( + id: [1L, 2L, 3L], date: [LocalDate.of(2026, 1, 1)] * 3, + text: ['first', 'second', 'third']).build() +MatrixParquetWriter.builder(events).pageSize(65536).write(file) -// Or write to a directory (filename derived from matrix name) -File dir = new File("output/") -MatrixParquetWriter.write(data, dir) // Creates output/cars.parquet +Matrix selected = MatrixParquetReader.builder().columns(['text', 'id']) + .filter(Predicates.ge('id', 2L)).limit(10).read(file) +assert selected.column('id') == [2L, 3L] ``` -### Builder API (recommended) +Projection accepts top-level fields in output order. Filter-only columns are read internally. +`Predicates.eq/ne/lt/le/gt/ge` have Number, String, Boolean, LocalDate, LocalDateTime, Instant, +LocalTime and byte[] overloads. Use `isNull`, `isNotNull`, `and`, `or` and `not` for composition. +Values must match the file's logical schema. Integer overflow, fractional integer values, +lossy decimal rescaling, and timestamp/time precision loss are rejected before reading pages. +Repeated/nested predicates are rejected. Missing indexes fall back to scans; corrupt data fails. + +String ordering uses unsigned UTF-8 bytes, matching Parquet. Unannotated BINARY predicates take +byte[] even when legacy metadata or `binaryAsString` makes output text. Comparisons with null are false; null tests are explicit. `not` is Boolean negation, including +for null rows. NaN comparisons are false except `ne`; numeric infinities retain their ordering. +Safe physical comparisons use row-group, dictionary, bloom and page-index pruning. Binary +DECIMAL comparisons and expressions without safe pushdown use exact residual evaluation. +LocalDateTime predicates on adjusted timestamps use `zoneId`; Instant predicates on naive +local timestamps are rejected. External TIME fields retain the existing `java.sql.Time` +millisecond output contract; filtering evaluates the original time precision. -The builder API provides a fluent, discoverable interface for reading and writing: +## Batches and shards ```groovy -import se.alipsa.matrix.core.Matrix import se.alipsa.matrix.parquet.MatrixParquetReader -import se.alipsa.matrix.parquet.MatrixParquetWriter -// Writing -MatrixParquetWriter.builder(matrix) - .precision(38) - .scale(18) - .zoneId('Europe/Stockholm') - .write(new File('output.parquet')) - -// Writing to byte array -byte[] bytes = MatrixParquetWriter.builder(matrix) - .inferPrecisionAndScale(true) - .writeBytes() - -// Writing to OutputStream -outputStream.withCloseable { os -> - MatrixParquetWriter.builder(matrix).write(os) -} - -// Reading -Matrix data = MatrixParquetReader.builder() - .matrixName('myData') - .zoneId('Europe/Stockholm') - .read(new File('data.parquet')) - -// Reading from byte array, InputStream, URL, or Path also supported -Matrix data = MatrixParquetReader.builder() - .read(parquetBytes) +MatrixParquetReader.builder().limit(1000).maxBatchRows(250) + .batches(new File('events.parquet')).withCloseable { batches -> + while (batches.hasNext()) { + def batch = batches.next() + assert batch.rowCount() <= 250 + } + } + +// Explicit lists retain caller order; directories and explicit globs sort normalized paths. +def dataset = MatrixParquetReader.builder().parallelism(2).sourceFileColumn('_source') + .read([new File('train-00001.parquet'), new File('train-00000.parquet')]) +def fromDirectory = MatrixParquetReader.read(new File('train')) +def fromGlob = MatrixParquetReader.readGlob('train/train-*.parquet') ``` -## Supported Data Types - -| Java/Groovy Type | Parquet Type | Notes | -|--------------------|--------------------------------|---------------------------------------------------| -| Integer | INT32 | | -| Long | INT64 | | -| BigInteger | BINARY (DECIMAL, scale=0) | Precision auto-inferred from data; no `Long` range limit | -| Float | FLOAT | | -| Double | DOUBLE | | -| BigDecimal | FIXED_LEN_BYTE_ARRAY (DECIMAL) | See [BigDecimal Precision](#bigdecimal-precision) | -| Boolean | BOOLEAN | | -| LocalDate | INT32 (DATE) | | -| java.sql.Date | INT32 (DATE) | | -| java.sql.Time | INT32 (TIME_MILLIS) | Millisecond precision; external TIME(MICROS/NANOS) is downgraded to milliseconds | -| LocalDateTime | INT64 (TIMESTAMP_MICROS) | Microsecond precision | -| java.sql.Timestamp | INT64 (TIMESTAMP_MICROS) | Microsecond precision | -| java.util.Date | INT64 | Epoch milliseconds | -| List | LIST | See [Nested Structures](#nested-structures) | -| Map | MAP or STRUCT | See [Nested Structures](#nested-structures) | -| Other | BINARY (STRING) | Stored as toString() | - -## BigDecimal Precision - -BigDecimal values require special handling to preserve precision. There are three approaches: - -### 1. Automatic Precision Inference (Default) -```groovy -// Automatically infers precision and scale from the data -MatrixParquetWriter.write(data, file) -// Or explicitly: -MatrixParquetWriter.write(data, file, true) -``` +A batch normally contains one selected row group. `maxBatchRows` splits large groups; individual +cells and compressed row-group pages still need heap. Close iterators on early exit; exhaustion +and errors close them automatically. Limits count accepted rows across batches/files. Footer +IDs remain valid when filters prune their groups; such groups produce no batch. There is no +ordinal row seeking API. -### 2. Uniform Precision for All BigDecimal Columns -```groovy -// Use precision=38, scale=18 for all BigDecimal columns -MatrixParquetWriter.write(data, file, 38, 18) -``` +Datasets validate every footer before returning rows and require identical logical fields, +order, decimal precision/scale, timestamp units/UTC flags and Matrix type metadata. Duplicate +canonical files and empty selections fail. Directory reads include direct `.parquet` children; +globs are explicit and can select deeper paths. Compatible Matrix indices are restored when every shard has the same index metadata and +projection retains its columns. A source column +contains canonical absolute paths and must not collide with a physical column. Parallel workers +use independent streams and converters, reuse each parsed footer, prefetch at most one batch per +worker, and preserve file/group order. `rowGroups` is for one file; use +`rowGroupsByFile: [(file.canonicalPath): [1, 0]]` for explicit dataset group selections. -Explicit precision/scale writes reject a value that would lose a non-zero fraction by default. To opt in to a deliberate rounding policy, use the builder or the typed options map: +## HTTP and Hub URIs ```groovy -import java.math.RoundingMode - -// Direct API -MatrixParquetWriter.write(data, file, 8, 2, RoundingMode.HALF_UP) - -// Builder API -MatrixParquetWriter.builder(data) - .precision(8) - .scale(2) - .roundingMode(RoundingMode.HALF_UP) - .write(file) +import se.alipsa.matrix.parquet.MatrixParquetReader +import se.alipsa.matrix.parquet.Predicates -data.write([precision: 8, scale: 2, roundingMode: 'HALF_UP'], file) +def remote = MatrixParquetReader.builder().remoteReadMode('AUTO').columns(['id']) + .filter(Predicates.ge('id', 1000L)).limit(100) + .readUrl('https://huggingface.co/datasets/owner/repo/resolve/main/train.parquet') +def hub = MatrixParquetReader.builder().remoteReadMode('AUTO').limit(20) + .readUrl('hf://datasets/owner/repo@refs/convert/parquet/train/0000.parquet') ``` -`roundingMode` accepts any `RoundingMode` enum name, case-insensitively. Use `UNNECESSARY` (the default) for lossless conversion; a rounded result must still fit the declared precision. +Replace owner/repo/path with a real dataset. DOWNLOAD preserves full-download behavior. AUTO +verifies `206 Content-Range` support then uses ranges, or downloads when the initial probe is +unsupported. RANGE requires range support. Once decoding starts, transport/object-change errors +propagate. Range caching is bounded by `rangeCacheBytes`; individual requests are capped separately at +8 MiB. Bulk reads request the needed span rather than splitting it into 64 KiB requests. Small +reads prefetch at most 64 KiB. Validated redirect targets are reused, and fully consumed range +responses permit HTTP keep-alive reuse. ETag/Last-Modified validators and total length detect +changes when available; same-length changes without validators cannot be detected reliably. +Redirects are bounded and HTTPS downgrades rejected. Connection failures before response headers +are retried once; malformed responses and body failures propagate. Streams are spooled to +temporary files; byte arrays use a seekable memory input. + +Hub dataset URI parsing is pinned to huggingface_hub v2.2.0. Revisions follow the repository: +`hf://datasets/owner/repo@revision/path`. Encode ordinary revision slashes as `%2F`; special +`refs/convert/` and `refs/pr/N` forms may remain unescaped. `@` inside a file path is literal. +Dot and parent path/revision segments are rejected. HF URI paths are literal (already-percent-encoded path text is escaped again). `HF_TOKEN` is +optional and read only for explicit hf:// calls; authorization is attached only to HTTPS +`huggingface.co` on its default/443 port and never to CDN redirect origins. This is HTTP range +access, with no Xet download/deduplication client. + +## Read options + +Use fluent builder methods, `MatrixParquetReader.builder(typedOptions)`, or SPI maps: +`Matrix.read([columns: ['id'], limit: 10L], file)`. Unknown options are rejected. Null map values preserve defaults, including nullable AUTO options. +`ParquetReadOptions.describe()` lists these options at runtime. + +| Option | Default | Meaning | +|---|---|---| +| matrixName | null | Explicit name; otherwise filename for local files, schema name for memory/streams/URLs | +| zoneId | system zone (null) | Adjusted timestamp conversion; ZoneId or zone name | +| columns | null/all | Nonempty unique ordered top-level output names | +| filter | null | Immutable Matrix-owned ParquetPredicate | +| binaryAsString | null/AUTO | Legacy Matrix unannotated binary is text; external/new binary is bytes; true reads external unannotated binary as UTF-8 recursively, false reads raw bytes; annotations and explicit new binary metadata take precedence | +| useColumnIndexFilter | true | Permit page pruning; false is useful for result/I/O comparisons | +| limit | null/unlimited | Nonnegative matching-row bound; zero retains schema | +| rowGroups | null/all | Unique, zero-based original footer IDs, caller order | +| rowGroupsByFile | empty/all | Dataset-only canonical file path to ordered group ID list | +| maxBatchRows | null | Positive accepted-row batch cap; default one row group | +| sourceFileColumn | null | Optional canonical local file/shard path column, added after projection; unsupported for memory/stream/URL sources | +| parallelism | 1 | Positive maximum dataset workers across files and row groups | +| remoteReadMode | DOWNLOAD | DOWNLOAD, AUTO or RANGE | +| connectTimeoutMillis | 10000 | Positive HTTP connection timeout | +| readTimeoutMillis | 60000 | Positive HTTP read timeout | +| rangeCacheBytes | 8388608 | Positive per-stream cache ceiling; independent request cap is 8 MiB | + +## Writer options and exact types -### 3. Per-Column Precision Specification ```groovy -// Specify precision and scale for specific columns -MatrixParquetWriter.write(data, file, [ - salary: [10, 2], // precision=10, scale=2 - rate: [8, 4] // precision=8, scale=4 -]) -``` - -**Note:** If the specified precision is too small for a value, an `IllegalArgumentException` will be thrown with a descriptive message indicating the required vs. allowed precision. - -## Nested Structures - -The module supports nested Groovy collections: +import se.alipsa.matrix.core.Matrix +import se.alipsa.matrix.parquet.MatrixParquetWriter -```groovy -def data = Matrix.builder('products').data( - id: [1, 2], - tags: [['electronics', 'sale'], ['furniture']], // List - attributes: [[color: 'red', size: 'L'], [color: 'blue']], // Map as STRUCT - reviews: [[[rating: 5, user: 'A']], [[rating: 4, user: 'B']]] // List of Maps -).types([Integer, List, Map, List]).build() - -MatrixParquetWriter.write(data, file) -Matrix restored = MatrixParquetReader.read(file) +def values = Matrix.builder('prices').columns(id: [1L, 2L], price: [1.25G, 2.50G]).build() +MatrixParquetWriter.builder(values).writerVersion('V2').compressionCodec('ZSTD') + .rowGroupSize(16L * 1024 * 1024).pageSize(65536) + .bloomFilterEnabled([id: true]).bloomFilterNDV([id: 1000L]) + .decimalMeta([price: [8, 2] as int[]]).write(new File('prices.parquet')) ``` -### Nested Structure Limitations -- Maps with heterogeneous value types are stored as STRUCT (preserves key names as field names) -- Maps with homogeneous value types are stored as MAP (standard Parquet map) -- Deeply nested structures are supported but may impact performance -- Bean/POJO objects are converted to Maps using reflection - -## Timezone Handling - -By default, `LocalDateTime` values are converted to/from UTC using the system default timezone. To use a specific timezone: - -```groovy -import java.time.ZoneId - -// Writing with a specific timezone -MatrixParquetWriter.write(data, file, ZoneId.of("America/New_York")) - -// Reading with a specific timezone -Matrix matrix = MatrixParquetReader.read(file, ZoneId.of("America/New_York")) - -// Reading with name and timezone -Matrix matrix = MatrixParquetReader.read(file, "myData", ZoneId.of("Europe/London")) +File, Path, byte-array and OutputStream writes share writer tuning. Size settings are targets, +not hard memory limits. V2 changes value encoders as well as page layout; V1 remains the default +for compatibility with older consumers. Bloom NDVs affect memory/false-positive tradeoffs and do +not enable a filter by themselves. Column settings use physical primitive paths (for example +`tags.list.element`); unknown/nonprimitive paths fail. Disabled bloom columns remain disabled. + +| Option | Default | Meaning | +|---|---|---| +| compressionCodec | SNAPPY | Codec name/CompressionCodecName; tested SNAPPY, ZSTD, GZIP, UNCOMPRESSED | +| inferPrecisionAndScale | true | Infer exact decimal settings when no explicit settings apply | +| precision / scale | null / null | Uniform positive precision and nonnegative scale, provided together | +| decimalMeta | empty | Logical field path to `[precision, scale]`; overrides uniform settings | +| roundingMode | UNNECESSARY | Reject lossy decimal rescaling; opt in to e.g. HALF_UP deliberately | +| zoneId | system zone (null) | LocalDateTime to adjusted timestamp conversion | +| rowGroupSize | 134217728 | Positive target row-group bytes | +| pageSize | 1048576 | Positive target data-page bytes | +| dictionaryEncoding | true | Dictionary encoding with upstream fallback | +| writerVersion | V1 | V1 or V2 writer encodings and page layout | +| columnIndexTruncateLength | 64 | Positive binary page-bound truncation bytes | +| statisticsTruncateLength | 2147483647 | Positive row-group binary statistics truncation bytes | +| bloomFilterEnabled | empty, disabled | Primitive path to Boolean override | +| bloomFilterNDV | empty | Primitive path to positive expected distinct-value count | +| sizeStatisticsEnabled | true | Collect size statistics | + +Files are written to a temporary sibling and atomically replace the destination after successful +close. Failed writes preserve an existing destination; filesystems without atomic replacement +support fail safely with `AtomicMoveNotSupportedException`. New files use normal process/umask +permissions; existing POSIX permissions are retained. Existing symbolic links are resolved before +writing, including dangling links, so the link is preserved and its target is created or replaced. +Abrupt JVM termination can leave `.matrix-parquet-.tmp` siblings beside the resolved output; +these are unpublished temporary output and can be removed once no writer is using them. + +JSON and ENUM annotations decode as UTF-8 String; BSON remains binary. +Strings are STRING-annotated recursively; byte[] stays raw BINARY. Byte/Short use signed width +annotations, UUID uses FIXED[16], LocalTime uses TIME(NANOS), and Instant uses adjusted +TIMESTAMP(MICROS). Instants with sub-microsecond precision or int64-microsecond overflow fail; +Instant.MIN/MAX are unsupported. Character and enums become String (`name()` for enums). +ZonedDateTime/OffsetDateTime/custom objects require explicit conversion to Instant/String/Map. +Date/Timestamp subclasses must share a single encoding policy; mixed units fail before output. + +Object and abstract/interface declarations scan every non-null value, check assignability and +infer one encoding; canonical metadata replaces the supertype. Concrete declarations require +instance-compatible values and a single runtime encoding policy. Mixed numeric classes need +explicit normalization. All-null Object becomes String; all-null Number/Temporal requires an +explicit supported concrete type. Containers use canonical List/Map metadata. All supported +metadata names are allowlisted; unknown external class names fall back to schema inference. + +Every resolved BigDecimal uses exact DECIMAL FIXED storage. Settings precedence is per-field, +uniform, then inference. Nested logical paths use `.element`, `.key`, `.value`, and struct field +names (for example `nested.element.amount`). With inference disabled every decimal needs settings. +All-null BigDecimal infers `(1, 0)`. There is no DOUBLE fallback. Explicit decimal conversion obeys +roundingMode. See [migration notes](release.md) for changed 0.7.x behavior. + +## CDC output, tests and benchmarks + +Native CDC writing is deferred: [parquet-java #3818](https://github.com/apache/parquet-java/pull/3818) +was open/unmerged when checked on 2026-10-17. Tuning pages/row groups is not CDC. For CDC output, +rewrite with PyArrow and preserve metadata: + +```bash +python3 -m venv /tmp/matrix-parquet-fixtures +/tmp/matrix-parquet-fixtures/bin/python -m pip install pyarrow==21.0.0 +/tmp/matrix-parquet-fixtures/bin/python - <<'PY' +import pyarrow.parquet as pq +table = pq.read_table('prices.parquet') +pq.write_table(table, 'prices-cdc.parquet', compression='zstd', + use_content_defined_chunking=True, write_page_index=True) +assert pq.read_table('prices-cdc.parquet').equals(table, check_metadata=True) +PY ``` -**Important:** For consistent round-trip behavior, use the same timezone for writing and reading. If files are shared across systems with different default timezones, explicitly specify the timezone. - -`java.sql.Time` is different: it is always treated as a local time of day in the JVM default timezone, independently of `zoneId`, and preserves milliseconds. External `TIME(MICROS)` and `TIME(NANOS)` values are read as `Time` with their sub-millisecond fraction truncated; the Parquet `isAdjustedToUTC` flag does not cause a zone shift. - -## Compression - -Parquet files are compressed with `SNAPPY` by default. Override per write: - -```groovy -import org.apache.parquet.hadoop.metadata.CompressionCodecName - -// Builder API -MatrixParquetWriter.builder(data) - .compressionCodec(CompressionCodecName.GZIP) - .write(file) - -// String form (also accepted) -MatrixParquetWriter.builder(data) - .compressionCodec('ZSTD') - .write(file) - -// SPI options map -data.write([compressionCodec: 'UNCOMPRESSED'], file) +This loads the whole Arrow table and adds a rewrite pass. Python is not a runtime dependency. +Hadoop filesystem configuration is avoided for local reads/writes; upstream codec initialization +still requires Hadoop classes, so the dependency and JDK 21 limit remain. + +```bash +./gradlew :matrix-parquet:codenarcMain :matrix-parquet:codenarcTest +./gradlew :matrix-parquet:spotlessCheck +./gradlew :matrix-parquet:test :matrix-parquet:compileConsumerApi +PARQUET_PYTHON=/tmp/matrix-parquet-fixtures/bin/python ./gradlew :matrix-parquet:test -PrunSlowTests=true +/tmp/matrix-parquet-fixtures/bin/python matrix-parquet/src/test/python/verify_matrix_output.py +./gradlew :matrix-parquet:test --tests ParquetInteropTest --rerun-tasks +./gradlew test ``` -Supported codec names: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `ZSTD`, `LZ4_RAW` (any value of `org.apache.parquet.hadoop.metadata.CompressionCodecName`). Compression is transparent on read — `MatrixParquetReader` auto-detects the codec from the file footer, no reader-side configuration is needed. - -## Known Limitations - -### Memory Usage -Matrix is an **in-memory data structure**. The entire Parquet file is loaded into memory when reading. Ensure you have sufficient RAM available for: -- The raw data size -- Java object overhead (typically 2-3x the raw data size) -- Working memory for processing - -For very large datasets that don't fit in memory, consider: -- Processing files in chunks externally before loading -- Using dedicated big data tools (Spark, Hadoop) for initial filtering -- Increasing JVM heap size (`-Xmx`) - -### JDK Compatibility -This module requires **JDK 21** and is limited to JDK 21 maximum due to Hadoop 3.4.x compatibility constraints. JDK 22+ is not supported. - -### Timestamp Precision -- `LocalDateTime` and `Timestamp` are stored with **microsecond** precision -- `Time` is stored with **millisecond** precision -- Nanosecond precision is not preserved - -### Type Metadata -Column type information is stored in Parquet file metadata. When reading files not created by `MatrixParquetWriter`: -- Types are inferred from Parquet schema logical type annotations -- Some type information may be lost (e.g., `java.sql.Timestamp` vs `LocalDateTime`) -- `BigInteger` columns read without Matrix metadata (e.g. external files) are inferred as `BigDecimal` with scale 0, since both the Parquet schema and the inference logic only see a generic `DECIMAL` logical annotation -- `BigInteger` values nested inside `List`/`Map` columns always get a declared precision of 38 regardless of their actual digit count (top-level column precision inference does not extend into nested structures), and round-trip back as numerically-equal `BigDecimal` values rather than `BigInteger` (nested elements carry no expected-type metadata, unlike top-level columns). This does not cause data loss or truncation, but external Parquet readers that validate `DECIMAL` precision against declared metadata rather than actual byte length may reject or misinterpret values with more than 38 digits - -## Technical Notes - -### Thread Safety -Both `MatrixParquetWriter` and `MatrixParquetReader` use thread-safe caching for: -- PropertyDescriptor lookups (for struct handling) -- Class.forName() results (for type parsing) - -The timezone parameter uses thread-local storage, making concurrent writes/reads with different timezones safe. - -### Performance Tips -1. Use `inferPrecisionAndScale=true` (default) for automatic BigDecimal handling -2. For large matrices with struct columns, the first write may be slower due to reflection caching -3. Subsequent operations on the same struct types benefit from cached metadata - -## 3rd Party Libraries - -| Library | Purpose | License | -|---------|---------|---------| -| org.apache.parquet:parquet-column | Parquet columnar format | Apache 2.0 | -| org.apache.parquet:parquet-hadoop | Hadoop integration | Apache 2.0 | -| org.apache.hadoop:hadoop-common | Hadoop utilities | Apache 2.0 | -| org.apache.hadoop:hadoop-mapreduce-client-core | MapReduce client | Apache 2.0 | +Fixture reproduction: run `matrix-parquet/src/test/python/generate_optimized.py` from the repository root with +pinned PyArrow. Checked-in Parquet fixtures total less than 4 MiB (each less than 1 MiB). +Default-size CDC is generated in `build/default-cdc` only in the slow lane; missing Python/Arrow +produces an explicit skip. CI provisions pinned Arrow and checks fixture reproducibility. +See [measurements and reproduction](benchmarks/optimized-parquet.md) for the opt-in benchmark. diff --git a/matrix-parquet/release.md b/matrix-parquet/release.md index 59fd65be9..25e394e0c 100644 --- a/matrix-parquet/release.md +++ b/matrix-parquet/release.md @@ -1,5 +1,66 @@ # Matrix-parquet Release History +## v0.8.0 (unreleased) + +Add CDC/page-index interoperability fixtures; a column-oriented row-group reader; typed predicates, +projection, limits and closeable batches; strict sharded datasets with bounded parallel decoding; +HTTP range reads; Hub dataset URI resolution; and shared writer tuning. Local files and byte arrays +use InputFile directly. New files carry `matrix.parquet.version=2`. + +Complete defaults, supported types and end-to-end examples are in [the module guide](readme.md). + +| Behavior | 0.8.0 result | Compatibility/opt-out | +|---|---|---| +| New Matrix strings | STRING-annotated BINARY recursively | No opt-out; standard annotation fixes interoperability | +| Legacy Matrix plain BINARY text | Remains String, including nested text/map keys | Automatic; metadata carrying the new format marker overrides legacy inference | +| 0.7.x byte[] columns | Fail with a contextual unrecoverable-legacy-binary error | The pinned 0.7.x writer fails with InvalidSchemaException for byte[]; legacy `[B` text payloads cannot recover bytes; regenerate from source | +| JSON / ENUM BINARY | UTF-8 String, including dictionary/null-only cases | Preserves pre-0.8 text behavior; BSON remains byte[] | +| External raw BINARY | byte[] | `binaryAsString(true)` restores prior UTF-8 behavior | +| Unadjusted timestamps | Local wall-clock time preserved | No bug-preserving flag; applications needing the former shift can explicitly convert the local value through UTC into the desired zone | +| Integer-backed DECIMAL | BigDecimal with declared scale | No bug-preserving flag; explicit `unscaledValue()` obtains prior raw numeric value | +| UUID annotation | UUID | No text-decoding compatibility flag; explicit UUID/string/byte conversion at application boundary | +| FLOAT16 / INT96 unsupported cases | Contextual unsupported-type error | No UTF-8 fallback; convert these types explicitly before reading | +| Byte / Short fallback | Signed INT32 with INT(8/16), restored as Byte/Short | Legacy unannotated text stays String; explicitly convert to String before writing to retain text output | +| LocalTime fallback | TIME(NANOS), restored as LocalTime using new-format metadata | Legacy text stays String; explicit String conversion retains text output | +| Instant fallback | TIMESTAMP(MICROS, adjusted=true), restored as Instant using new-format metadata | Legacy text stays String; reject sub-microsecond precision loss and int64-micros overflow; explicitly convert to String for text output | +| UUID writer fallback | FIXED[16] + UUID, restored as UUID | Legacy text stays String; explicit String conversion retains text output | +| Character fallback | STRING, returned as String | Normalize emitted metadata to String; Character identity is not preserved | +| Enum fallback | STRING using enum name(), returned as String | Normalize metadata to String; convert explicitly for former overridden toString() output | +| ZonedDateTime / OffsetDateTime fallback | Explicit unsupported-type error | Convert to Instant to retain instant, or String to retain zone/offset text; no implicit zone loss | +| Object column, single supported policy type | That type's encoding and canonical metadata | Reads back typed as the inferred canonical class instead of Object; no explicit types required | +| Abstract/interface declared supertype (Number, CharSequence, Comparable, Serializable, Temporal) | Validate assignability, infer one supported policy type and emit its canonical name | Reads back as the resolved class instead of the declared supertype; normalize heterogeneous values explicitly | +| Concrete declared type with incompatible runtime values | Validation error before writing | Box primitives then require every non-null value to be an instance; no silent schema/metadata mismatch | +| BigDecimal, declared or inferred through a supertype | Exact FIXED_LEN_BYTE_ARRAY DECIMAL with resolved precision/scale | Remove DOUBLE fallback; infer by default, or require explicit decimal settings when inference is disabled; no lossy compatibility mode | +| All-null Number/Temporal declaration | Validation error before writing | New write failure: retype explicitly to a supported concrete type; no nominal String metadata | +| Concrete Date column mixing Date/Timestamp or sql.Date/Timestamp | Validation error before writing | Runtime encoding policies differ; normalize every value to one supported temporal class before writing | +| All-null Object column | STRING/BINARY with canonical String metadata | Retain current text schema behavior; reads back as String-typed column | +| Mixed/unsupported Object column or other custom types | Contextual validation error before writing | Name the first offending row/class; explicitly normalize values or map to String | + +Unknown metadata class names are never loaded and fall back to physical/logical schema inference. +Incompatible metadata declarations also fall back rather than creating falsely typed columns. +Legacy Matrix text compatibility applies recursively; new raw binary metadata remains authoritative. +The checked-in original 0.7.x string fixture predates these writer changes. Legacy byte[] recovery +claims were narrowed after an end-to-end check found InvalidSchemaException in this checkout. + +External INT96/FLOAT16 raise contextual unsupported-type errors. External TIME still returns +java.sql.Time at millisecond precision; exact predicate filtering preserves original precision. +Mixed Date/Timestamp policies are now rejected before writing, preventing the prior 1000× unit +corruption. New Object/supertype values are resolved consistently for schema and metadata. +Unknown SPI options are now rejected. Dataset reads restore common compatible Matrix indices, require matching type +metadata, and reject empty selections and duplicate paths. + +Review fixes also cover empty Arrow dictionaries; consistent static URL/Hub and directory overloads; +local source columns; daemon dataset workers; null option-map defaults; path-segment validation; +projection around legacy `[B` columns; one validated type-policy scan per column; and atomic output +replacement on success. New files use normal process/umask permissions; existing POSIX permissions +are retained, and writes through existing symbolic links update the resolved target while preserving +the link. Local file writes require same-directory atomic replacement support; unsupported network +or FUSE filesystems raise `AtomicMoveNotSupportedException` and leave existing output unchanged. +HTTP bulk requests reuse resolved targets and a separately bounded cache. + +Native CDC writing remains deferred to upstream parquet-java #3818; the Python rewrite preserves +custom metadata. Hadoop stays as a runtime dependency because codec initialization still loads it. + ## v0.7.0, 2026-09-22 - SPI `inferPrecisionAndScale` parsing is now strict: Boolean values and trimmed, case-insensitive `true`/`false` strings are accepted; former truthy inputs are rejected. - Breaking change: explicit DECIMAL scale conversion now defaults to `RoundingMode.UNNECESSARY`, rejecting lossy conversion with a contextual `IllegalArgumentException`. Set `roundingMode: 'HALF_UP'` (or another named `RoundingMode`) when rounding is intentional. diff --git a/matrix-parquet/req/optimized-parquet-plan.md b/matrix-parquet/req/optimized-parquet-plan.md new file mode 100644 index 000000000..5429af520 --- /dev/null +++ b/matrix-parquet/req/optimized-parquet-plan.md @@ -0,0 +1,786 @@ +# Hugging Face optimized Parquet: analysis and enhancement plan + +Analysis date: 2026-10-17. Scope: compatibility and baseline measurements (P0), columnar and bounded local reads, +typed predicates and writer tuning (P1), sharded datasets and HTTP ranges (P2), +optional Hub resolution and upstream CDC tracking (P3). Implementation and verification evidence is recorded at the end of this document. +The findings below describe the pre-implementation baseline. + +## Findings + +### What “optimized Parquet” means + +Hugging Face describes optimized Parquet as ordinary Parquet written with content-defined +chunking (CDC) and page indexes. These are writer/layout features, not a separate container +format, codec, file extension, or reader mode. PyArrow exposes +`use_content_defined_chunking=True` and `write_page_index=True`; its CDC option requires +PyArrow 21 or later. [Hugging Face documentation](https://huggingface.co/docs/hub/en/datasets-libraries), +[PyArrow integration](https://huggingface.co/docs/hub/en/datasets-pyarrow). + +CDC chooses boundaries from logical column content before encoding/compression. Readers +decode the resulting ordinary pages without reproducing the chunking algorithm. Xet can +reuse matching stored chunks across revisions; reading a downloaded local file does not +itself invoke Xet or give matrix-parquet incremental download capability. +[Hugging Face CDC explanation](https://huggingface.co/blog/parquet-cdc). + +Page indexes consist of `ColumnIndex` (page statistics) and `OffsetIndex` (page locations +and first-row positions), associated with column chunks. They support skipping candidate +pages and aligning reads across columns. Indexes are optional, and page boundaries need +not align across columns. Filtering must still reject nonmatching records within candidate +pages. [Parquet page-index specification](https://parquet.apache.org/docs/file-format/pageindex/). + +PyArrow can move page statistics from page headers to the index when index writing is +enabled. Thus tests must include real PyArrow output rather than assuming that Java-written +round trips cover this layout. CDC and page indexes are independent switches; test all four +combinations. Do not assume fixed page sizes or infer CDC from a page-size distribution. +[PyArrow writer reference](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.write_table.html). + +### Pre-implementation repository behavior + +The authoritative dependencies are **parquet-java 1.18.1**, Hadoop **3.5.0** in +`gradle/libs.versions.toml`; the module is **0.7.1-SNAPSHOT**. The older v0.6.0 plan is +historical and should not determine dependency decisions here. Keep JDK 21. + +| Area | Evidence in current code | Assessment | +|---|---|---| +| Full-file decoding | `MatrixParquetReader.read(File, ...)` reads the footer and uses `ParquetReader` / `GroupReadSupport` | CDC/indexed files should decode through the existing dependency; unproven until external fixtures run | +| Selective reads | `ParquetReadOptions` and `ReaderBuilder` only expose name/time zone; core loop materializes every row and field | No caller-supplied predicate, projection, or bounded read; no useful page pruning for the current full scan | +| Remote reads | URL overloads open a stream; `readFromInputStream` copies it entirely to a temporary file | Entire file transfers before reading; page pruning cannot reduce network traffic | +| Metadata | Reader uses Matrix-specific type/index keys when present, otherwise schema inference | External files need tests without Matrix metadata; Matrix indexes are unrelated to Parquet page indexes | +| Writes | File and memory writers configure schema, codec and Matrix metadata | Index production is delegated to upstream defaults; no CDC configuration | +| Coverage | Six Groovy test classes and `mtcars.parquet` fixture; no CDC/index-specific suite | Cross-implementation and page-pruning coverage missing | + +parquet-java 1.18.1 already exposes `withFilter`, `useColumnIndexFilter(boolean)`, and +record filtering. Column-index filtering is enabled by default by its read-options builder. +The gap is supplying a predicate and preserving correct materialization, rather than merely +turning an upstream flag on. +[Reader API](https://github.com/apache/parquet-java/blob/apache-parquet-1.18.1/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetReader.java), +[read defaults](https://github.com/apache/parquet-java/blob/apache-parquet-1.18.1/parquet-hadoop/src/main/java/org/apache/parquet/ParquetReadOptions.java). + +The upstream writer enables statistics by default and serializes column/offset indexes. +Consequently, “add page-index writing” is not yet an established missing feature: first +inspect actual matrix-parquet output for supported columns. There is no CDC option in the +examined 1.18.1 `ParquetWriter.Builder` or `ParquetProperties.Builder`. Do not introduce a +nonfunctional `optimizedParquet` or `contentDefinedChunking` option. +[Writer API](https://github.com/apache/parquet-java/blob/apache-parquet-1.18.1/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java), +[writer properties](https://github.com/apache/parquet-java/blob/apache-parquet-1.18.1/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java), +[index serialization](https://github.com/apache/parquet-java/blob/apache-parquet-1.18.1/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java). + +### Public API and dependency boundary + +`parquet-column` and `parquet-hadoop` are `implementation` dependencies. Exposing +`FilterPredicate` would force downstream consumers to add parquet-java to their compile +classpath or require changing publication scope to `api`. Use Matrix-owned predicate and +writer-version types instead; translate them internally. Keep parquet-java implementation +private for the new API and add downstream Gradle/Maven compile smoke tests. Existing public +codec signatures are a separate legacy exposure to characterize, not a reason to introduce +more upstream types. + +### Adjacent schema risks + +These affect Hugging Face datasets independently of optimization and must be tracked +separately from CDC compatibility: + +- Unannotated `BINARY`/`FIXED_LEN_BYTE_ARRAY` currently defaults to `String`, and + `readPrimitive` decodes it as UTF-8. Arbitrary image/audio bytes therefore lose fidelity. +- DECIMAL annotations on `INT32`/`INT64` are inferred as `BigDecimal`, but the corresponding + primitive read branches return raw integers without applying decimal scale. +- TIMESTAMP decoding ignores `isAdjustedToUTC=false` and shifts local wall-clock values + through the configured/system zone. Test naive PyArrow timestamps explicitly before declaring + their annotation confirmed by fixtures. +- UUID and FLOAT16 annotations on FIXED_LEN_BYTE_ARRAY reach the UTF-8 fallback. INT96 + reaches `group.getString`; the exact exception remains to be characterized by a fixture. Neither + path constitutes valid decoding. +- `readList` expects the standard three-level `list` structure. Include PyArrow compliant + lists, nested structs and null/empty values; do not claim support for all historical LIST + representations without additional work. + +Prefer a small representative text dataset as the initial compatibility target. Include +binary/media and integer-backed decimal fixtures to make broader limitations explicit. + +## Implementation sequence + +### 0. Set the release and migration contract (P0 prerequisite) + +- 0.1 [x] Before production changes, set `matrix-parquet/build.gradle` to `0.8.0-SNAPSHOT`; target + release **0.8.0**. Keep P2/P3 follow-ups separately releasable rather than promising their + delivery in 0.8.0. Add a 0.8.0 migration section to release notes with the behavior table below, + complete option defaults and runnable examples. + +- 0.2 [x] Capture representative 0.7.x-produced fixtures before changing the writer, including + top-level/nested strings, string map keys and Matrix temporal/decimal metadata. Include a 0.7.x + byte[] failure fixture; do not assume every old file is recoverable. Include them in the 4 MiB + fixture budget. Add before/after characterization tests plus external fixtures for behavior + 0.7.x did not write, asserting values and runtime types. The Group reader retained for + differential tests must receive phase-2 fixes; separately preserve original expected 0.7.x + results for migration characterization. + + | Behavior | 0.8.0 result | Compatibility/opt-out | + |---|---|---| + | New Matrix strings | STRING-annotated BINARY recursively | No opt-out; standard annotation fixes interoperability | + | Legacy Matrix plain BINARY text | Remains String, including nested text/map keys | Automatic; metadata carrying the new format marker overrides legacy inference | + | 0.7.x byte[] columns | Fail with a contextual unrecoverable-legacy-binary error | The pinned original writer fails with InvalidSchemaException for byte[]; legacy `[B` text cannot recover bytes; regenerate from source | + | External raw BINARY | byte[] | `binaryAsString(true)` restores prior UTF-8 behavior | + | Unadjusted timestamps | Local wall-clock time preserved | No bug-preserving flag; applications needing the former shift can explicitly convert the local value through UTC into the desired zone | + | Integer-backed DECIMAL | BigDecimal with declared scale | No bug-preserving flag; explicit `unscaledValue()` obtains prior raw numeric value | + | UUID annotation | UUID | No text-decoding compatibility flag; explicit UUID/string/byte conversion at application boundary | + | FLOAT16 / INT96 unsupported cases | Contextual unsupported-type error | No UTF-8 fallback; supported-type contract fixed before release | + | Byte / Short fallback | Signed INT32 with INT(8/16), restored as Byte/Short | Legacy unannotated text stays String; explicitly convert to String before writing to retain text output | + | LocalTime fallback | TIME(NANOS), restored as LocalTime using new-format metadata | Legacy text stays String; explicit String conversion retains text output | + | Instant fallback | TIMESTAMP(MICROS, adjusted=true), restored as Instant using new-format metadata | Legacy text stays String; reject sub-microsecond precision loss and int64-micros overflow; explicitly convert to String for text output | + | UUID writer fallback | FIXED[16] + UUID, restored as UUID | Legacy text stays String; explicit String conversion retains text output | + | Character fallback | STRING, returned as String | Normalize emitted metadata to String; Character identity is not preserved | + | Enum fallback | STRING using enum name(), returned as String | Normalize metadata to String; convert explicitly for former overridden toString() output | + | ZonedDateTime / OffsetDateTime fallback | Explicit unsupported-type error | Convert to Instant to retain instant, or String to retain zone/offset text; no implicit zone loss | + | Object column, single supported policy type | That type's encoding and canonical metadata | Reads back typed as the inferred canonical class instead of Object; no explicit types required | + | Abstract/interface declared supertype (Number, CharSequence, Comparable, Serializable, Temporal) | Validate assignability, infer one supported policy type and emit its canonical name | Reads back as the resolved class instead of the declared supertype; normalize heterogeneous values explicitly | + | Concrete declared type with incompatible runtime values | Validation error before writing | Box primitives then require every non-null value to be an instance; no silent schema/metadata mismatch | + | BigDecimal, declared or inferred through a supertype | Exact FIXED_LEN_BYTE_ARRAY DECIMAL with resolved precision/scale | Remove DOUBLE fallback; infer by default, or require explicit decimal settings when inference is disabled; no lossy compatibility mode | + | All-null Number/Temporal declaration | Validation error before writing | New write failure: retype explicitly to a supported concrete type; no nominal String metadata | + | Concrete Date column mixing Date/Timestamp or sql.Date/Timestamp | Validation error before writing | Runtime encoding policies differ; normalize every value to one supported temporal class before writing | + | All-null Object column | STRING/BINARY with canonical String metadata | Retain current text schema behavior; reads back as String-typed column | + | Mixed/unsupported Object column or other custom types | Contextual validation error before writing | Name the first offending row/class; explicitly normalize values or map to String | + +### 1. Establish compatibility and writer-index facts (P0, first deliverable) + +- 1.1 [x] Add `OptimizedParquetTest.groovy` and checked-in deterministic fixtures + under `src/test/resources/optimized/`. Add a generator script plus a manifest containing + generator/PyArrow version, options, schema, row counts, expected values and SHA-256 checksums. + Python is a fixture-generation tool, not a runtime or normal Gradle-test dependency. + +- 1.2 [x] Generate paired ordinary/CDC-only/index-only/CDC+index files from identical values. + Include SNAPPY and ZSTD, dictionary enabled/disabled, data pages v1/v2, multiple pages per + column and multiple row groups. Use deterministic varied long strings so the CDC fixtures + actually cross chunk boundaries; verify physical multi-page structure. Set a **4 MiB aggregate + checked-in binary-fixture budget**, with a 1 MiB per-file cap. Use CDC `{'min_chunk_size': 4096, + 'max_chunk_size': 16384, 'norm_level': 0}` for small fixtures, and verify the resulting + boundaries/pages. PyArrow 21 accepts this dictionary with both size keys required, confirmed in + its [versioned implementation](https://github.com/apache/arrow/blob/apache-arrow-21.0.0/python/pyarrow/_parquet.pyx). + Add an opt-in default-CDC generation/interoperability test under `./gradlew :matrix-parquet:test + -PrunSlowTests=true`, using a build-directory fixture. Skip with an explicit reason if Python or + pinned PyArrow is absent; do not silently install packages. Add the Python/PyArrow lane to + `.github/workflows/release-all-tools.yml` (including its pull_request runs), invoking the opt-in + command so this case actually runs. Fail generation errors when dependencies are available and + enforce the checked-in budget. Pairwise coverage replaces a Cartesian product. Include + ordered/unordered columns, null-only pages, null/empty nested lists and structs, Unicode, + decimals and timestamps. + +- 1.3 [x] Assert exact values, types, order and null semantics through file, path, bytes, stream, + builder and SPI entry points. Use a local HTTP server for URL paths; regular tests must not + depend on Hugging Face availability or credentials. Cover empty files and external metadata + without Matrix-specific keys. + +- 1.4 [x] Inspect index references and deserialize `ColumnIndex`/`OffsetIndex` using + `ParquetFileReader`. Check page counts, monotonic offsets, row positions and valid bounds. + Inspect both Java-produced file and byte-array output. Record which types produce indexes and + any upstream omissions. Do not infer index presence from a flag. + +- 1.5 [x] Repair any demonstrated CDC/index decoding failure with a targeted regression test. If + all supported-schema fixtures pass, record that no CDC decoder change is necessary. Do not + upgrade parquet-java solely because the files use CDC. + +- 1.6 [x] Before reader changes, establish a repeatable full-scan baseline using generated + approximately 1M-row wide-text, low-cardinality categorical and mixed/nested datasets. Keep + large inputs outside Git. Record generation commands/seeds, JVM/heap/GC, CPU, warmup and + repeated-run medians, rows/s, elapsed time, allocation/GC, peak heap and bytes read. Separate + cold and warm I/O and compare ordinary/indexed/CDC+indexed layouts. Reuse these exact inputs + throughout P1/P2; include narrow projection and selective predicates as soon as those APIs + exist. + +### 2. Fix demonstrated external-schema fidelity gaps (P0 for broad HF support) + +Order within 2.0: **2.0 → 2.0c (shared descriptor) → 2.0a/2.0b/2.0d +(policies implemented on that descriptor) → 2.0e (integration verification)**. +Task identifiers remain stable for review references; their numeric order does not override +these dependencies. Do not implement policies on the old first-value inference as an interim step. + +- 2.0 [x] Before 2.1, write String as BINARY + `LogicalTypeAnnotation.stringType()` through the + shared primitive schema builder, including list elements, struct fields, map values and required + map keys. Verify the annotations with PyArrow. Preserve existing files: when Matrix metadata + identifies a legacy file, unannotated BINARY remains String recursively, including nested values + whose expected type is null. New binary-capable output must include a versioned Matrix + schema/encoding marker to distinguish raw binary from legacy nested text; do not apply legacy + rules blindly to newly written media bytes. Add nullable Boolean `binaryAsString` (default + null/AUTO): null uses legacy-Matrix text compatibility and raw bytes for external/new-format + files; true forces unannotated BINARY to UTF-8 strings recursively; false forces raw bytes + recursively. Annotations take precedence; explicitly declared Matrix String fields remain + strings. Define FIXED binary separately: never override DECIMAL/UUID/FLOAT16 annotations. + Propagate through options, builder and SPI, and test old nested strings/map keys plus new mixed + binary/text fields. + +- 2.0a [x] Require raw byte[] writing as BINARY without STRING annotation, with the new format + marker, at top level and inside nested structures. Add an explicit byte[] branch using Parquet + Binary bytes; reject unsupported schema/value combinations with contextual errors instead of the + `value.toString()` fallback only after implementing the explicit type policies in 2.0b–2.0d. Land + 2.0a–2.0e together so fallback removal has no implicit scope. Resolve metadata array class names + through a fixed allowlist mapping of names to supported writer-produced types, including `[B` to + byte[]. Do not load arbitrary classes named by untrusted file metadata. If a resolver still + needs `Class.forName(name, false, loader)`, call it only for allowlisted names. Unknown names + fall back to per-field schema inference, without failing the whole read; malformed metadata must + not shift type/column alignment. Test unknown/custom class names and arrays. The legacy `[B` + corruption rule below takes precedence over both allowlist resolution and schema fallback. Add + file/bytes/stream/SPI round-trip tests for null, empty and arbitrary non-UTF-8 bytes, plus + PyArrow verification of exact payloads. Characterize original byte[] write failure and the `[B` + class-resolution failure. Use explicitly synthetic legacy text metadata if the original writer + cannot produce a valid file; fixing class lookup cannot recover a textual payload. Fail explicitly on legacy Matrix byte[] metadata rather than returning the + textual payload as if it were original binary. State this in migration notes and test it; + marker-aware new byte[] files must read normally. + +- 2.0b [x] Implement per-type encodings and explicit rejections. Replace remaining implicit + `toString()` cases with the policies in 0.2. Implement Byte/Short as signed INT32 + + `intType(8/16, true)`, with range validation and annotated nested-width inference. Encode LocalTime as + TIME(NANOS) to preserve nanoseconds, and Instant as adjusted TIMESTAMP(MICROS). Reject Instant + values with sub-microsecond precision rather than silently truncating, and validate int64 + epoch-micros overflow using checked arithmetic, including negative/pre-epoch instants. Test + historical dates beyond the int64-nanos range, exact microseconds, fractional-microsecond + rejection and extreme Instant bounds. MICROS has a wider range than NANOS but still cannot + represent Instant.MIN/MAX; do not promise those. No NANOS write option is required in this + release. New-format metadata restores these exact Java types; document external schema-inference + types separately and preserve existing java.sql.Time/LocalDateTime contracts. Encode UUID as 16 + bytes with `uuidType()` in the specification's byte order. Character becomes STRING; enums become + STRING using `name()`, with canonical String metadata at every supported nesting level. Reject + ZonedDateTime, OffsetDateTime and remaining custom classes before writing rows; messages describe + explicit Instant/String conversion or column retyping. + +- 2.0c [x] Unify column type resolution and validation. Resolve every column through one validated + schema/type descriptor used for physical schema, value encoding, decimal settings and emitted + metadata. For Object and abstract/interface supertypes (Number, CharSequence, Comparable, + Serializable, Temporal), scan all non-null values, require assignability to the boxed declaration + and one supported policy type, then emit that encoding's canonical name. Never decide from only + the first sample. For concrete declarations, including non-final concrete classes, require each + non-null value to be an instance after primitive boxing normalization; additionally validate the + single encoding policy across subclasses. A non-final modifier alone does not turn a concrete + declaration into an inference-only supertype. Report the first offending zero-based row, declared + type, runtime class and expected policy before output mutation. Resolve supported concrete + all-null columns from their declaration; Object or abstract/interface all-null columns use + canonical String only if compatible with the declaration, otherwise require an explicit supported + type. Thus all-null Number/Temporal columns need explicit retyping rather than false String + metadata. For supported concrete declarations, the single encoding policy comes from the runtime + values, while metadata retains the declared supported type unless an explicit canonicalizing + policy (Character/Enum) applies. Empty columns use the declared policy. Mixed Date/Timestamp + values fail even though Timestamp is assignable to Date, because MILLIS/MICROS policies differ; + never dispatch incompatible units per value. Add a legacy characterization test for Date first, + Timestamp later, recording the current 1000× timestamp corruption, and sql.Date first, Timestamp + later, recording the current write failure. New tests require rejection before output mutation in + either order, plus homogeneous temporal-subclass round trips with declared Date metadata. Test + Object/supertype inference, concrete mismatch, boxed normalization and all-null rules separately. + Test Number/CharSequence/Comparable/Serializable/Temporal declarations, and reject a concrete + BigDecimal column containing Integer values before writing. + +- 2.0d [x] Route every resolved BigDecimal column, declared or resolved from Object/Number/other + supertypes, through shared `inferDecimalPrecisionAndScale` and column-name `decimalMeta` handling + before schema construction. Use explicit per-column settings first, explicit global + precision/scale next, then inference if enabled. With `inferPrecisionAndScale(false)`, require + complete explicit settings for each resolved BigDecimal field; reject missing settings rather than + falling back to DOUBLE. Apply existing rounding/precision validation to all these paths. Always + emit FIXED_LEN_BYTE_ARRAY + DECIMAL and canonical BigDecimal metadata together; remove the DOUBLE + fallback for declared BigDecimal too. Handle nested decimals with the same resolved-field policy; + document nested metadata addressing and reject unresolved settings. Use a documented minimum + precision/scale (1, 0) for all-null declared BigDecimal when inferring. Test untyped Groovy + literals `[1.5, 2.25]`, high-precision values that DOUBLE cannot represent, negative scales, + Number-declared decimals, per-column overrides and inference disabled with and without explicit + settings. Assert exact BigDecimal values, runtime classes, declared Matrix types, physical DECIMAL + metadata and all-null behavior across entry points. General declared-type validation tests belong + to 2.0c. Canonical metadata must never claim BigDecimal over a DOUBLE or INT32 encoding. + +- 2.0e [x] Verify metadata coverage, migration contracts and cross-entry-point behavior. Write the + canonical type name instead of Object or the abstract/interface declaration. Canonicalized + Character/Enum values follow the String policy; do not silently widen heterogeneous numeric types. + Reject mixed or unsupported policy types with the first offending row index (document zero-based), + actual class and expected policy. All-null Object columns retain STRING/BINARY with canonical + String metadata. Validate nested inferred schemas consistently before writing. Do not discover + unsupported types halfway through overwriting an output file. Include null/all-null columns and + boxed primitive normalization in schema validation. Add ordinary/optimized and file/bytes/SPI + round-trip tests for each supported policy, nested variants where supported, and contextual + rejection tests for each rejected class. Characterize legacy text separately: it remains String, + not reconstructed Java objects. Assert writer-emitted matrix.columnTypes names are a subset of the + resolver allowlist across all supported types and canonicalized columns, including Byte/Short and + containers. Never emit java.lang.Object or arbitrary enum/custom class names in new files; old + unknown names continue to use the schema fallback. Test the untyped builder example with + Integer/String data, homogeneous nullable Object columns, all-null Object columns and mixed + `[1, 'x']` rejection before output mutation. Assert inferred canonical metadata names and resulting + Matrix types, including Object columns of each supported policy type. Extend migration notes with + each 0.2 row. For each of 2.0a–2.0e, run the ordered module checks in 6.3 and record the + successful commands/results before checking off that task; do not use a single completion checkbox + for all five areas. + +- 2.1 [x] Add byte-preserving inference/conversion for raw binary fields, including binary inside + nested media structs. Retain string decoding for STRING annotations and DECIMAL handling and the + 2.0 legacy/override rules. Document the compatibility impact of changing external raw-binary + columns from String to byte[]. Binary writer/reader round-trip support in 2.0a is mandatory, not + conditional. + +- 2.2 [x] Apply DECIMAL scale to INT32/INT64 physical values, including nested values; test + negative, null, scale-zero and fractional values against PyArrow expectations. + +- 2.3 [x] Respect TIMESTAMP `isAdjustedToUTC`: adjusted timestamps use the configured zone; + unadjusted timestamps retain local wall-clock fields via UTC epoch arithmetic + (`LocalDateTime.ofEpochSecond(instant.epochSecond, instant.nano, ZoneOffset.UTC)`). Cover + MILLIS/MICROS/NANOS, pre-epoch values, nested timestamps, DST boundaries and + UTC/Europe-Stockholm/America-New_York. Generate naive and timezone-aware PyArrow fields and + assert their footer annotations; preserve adjusted Matrix round trips. Specify Date/Timestamp + metadata overrides explicitly rather than silently assigning an instant to local timestamps. + +- 2.4 [x] Add UUID, FLOAT16 and INT96 fixtures. Decode UUID to `UUID`; define FLOAT16 widening to + `Float` with special-value tests, or fail explicitly if unsupported in the first delivery. + Characterize INT96 failure and implement documented legacy timestamp semantics or a contextual + unsupported-type error. Never silently decode these as text. Cover compliant, two-level and + legacy LIST layouts with explicit support/error contracts. + +- 2.5 [x] Keep these tests paired across ordinary and optimized layouts so failures are attributed + to schema conversion rather than CDC. Report unsupported logical types explicitly; broaden + beyond these cases only when a fixture demonstrates need. + +### 3. Expose selective local reads using upstream page filtering (P1) + +- 3.1 [x] Build a shared `InputFile` core using `LocalInputFile` and `PlainParquetConfiguration`; + use `LocalOutputFile`/`OutputFile` and plain configuration on file writes too. Preserve naming, + time-zone and ownership behavior across legacy, builder and SPI paths. Add an in-memory seekable + `InputFile` for byte arrays, replacing their temp-file copy. Test seeks, EOF and independent + streams without full-buffer copies. Use `ParquetFileReader.open(InputFile, ParquetReadOptions)` + and an explicit per-row-group loop, with `getRowGroups()`, + `readFilteredRowGroup(int)`/`readRowGroup(int)` and `MessageColumnIO.getRecordReader(pages, + materializer, filter)`. Set requested schema before filtering. Keep original footer group IDs + stable even if pruning removes groups from an internal list; spike and test this mapping before + exposing selectors. Close each `PageReadStore` on advancement/failure. Do not build the new core + around `ParquetReader`. Its 1.18.1 API does expose current-row-group index, but does not provide + direct indexed group selection; explicit groups fit batching and independent workers better. + Follow the upstream [internal reader loop](https://github.com/apache/parquet-java/blob/apache-parquet-1.18.1/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordReader.java), + without depending on that package-private class. Keep accept/rollback bookkeeping in the Matrix + materializer using the hooks in 3.2; the loop respects skipped/null records without duplicating + column-list truncation. Spike this integration before removing Hadoop paths. Test + SNAPPY/ZSTD/GZIP/uncompressed read/write with hadoop-common/mapreduce excluded, and inspect + linkage/dependency reports. Keep required codec classes until this proves removal safe; + distinguish avoiding filesystem initialization from removing dependencies. Do not raise the JDK + target as part of this change. + +- 3.2 [x] Replace `GroupReadSupport` with a custom `ReadSupport`/`RecordMaterializer` whose + converters append to typed column lists. Precompute schema/converter mappings and avoid per-cell + name lookup, per-row Group/list/Object[] creation and row-to-column transposes. Spike Matrix + construction from columns to verify ownership/copy behavior. Preserve nested list/map/struct + nulls, empty collections and parent-row alignment. Any active row filter, including ordinary + pushed-down record filtering, requires transactional output: converters may append values before + acceptance is known. Snapshot each output/filter-column list size at row start. Accept in Matrix + `RecordMaterializer.getCurrentRecord()`, which the filtering wrapper calls only after its + pushed-down predicate passes: evaluate any residual predicate there before accepting and return + null on rejection. Return a non-null lightweight record token for acceptance, including all-null + rows. Roll back to the snapshots in `skipCurrentRecord()`, reached through parquet-java's + skipped/null-record handling, and reset nested assembly state. Keep rollback idempotent. Mid-row + exceptions must invoke the same materializer-owned abort/rollback operation before propagating; + snapshots also protect this path. Test the actual hook ordering with pushed-down and residual + rejection. Test rejected first/middle/last rows and entire rejected groups, ensuring every + column has the same accepted-row count. Use staging where nested assembly requires it, without + allocating a generic row buffer for every unfiltered flat row. Differential-test against the old + reader before adding new projection/filter options; preserve schema fixes from phase 2 in shared + conversions. A ReadSupport adapter may initialize the converter tree, but the explicit + ParquetFileReader loop owns group selection and requested schema. + +- 3.3 [x] Implement dictionary-aware string reuse in converters: decode each dictionary ID once + per active dictionary and reuse the String reference, supporting dictionary changes and + plain-page fallback. No global `String.intern()` or unbounded cross-file cache. Verify identity + reuse on categorical data and exact values after dictionary fallback. + +- 3.4 [x] Add the proposed options below, including `toMap`, `fromMap`, descriptors, validation, + builder methods and public GroovyDoc. Use Matrix-owned immutable predicate types, translated + internally; no upstream predicate classes or closures in the new public contract. Reject unknown + columns, duplicates and incompatible predicate types before reading. Start projection with + top-level column names. + + | Proposed option | Type | Default | Meaning | + |---|---|---|---| + | `columns` | `List` | `null` | All columns; supplied nonempty list selects output columns in requested order | + | `filter` | Matrix-owned `ParquetPredicate` | `null` | Logical-schema predicate with exact record filtering | + | `useColumnIndexFilter` | `boolean` | `true` | Allow upstream page pruning when available; false is a comparison/diagnostic path | + +- 3.5 [x] Pass translated upstream predicates via `FilterCompat.get(predicate)`. The core loop + sets the union of output columns and all pushdown/residual filter columns via + `ParquetFileReader.setRequestedSchema(MessageType)` before filtering/group reads. ReadSupport + only initializes the matching converter/materializer tree; it does not own a second projection + implementation. Keep exact record filtering enabled. Read filter-only columns internally and + exclude them from the resulting Matrix. Add `Predicates.eq/ne/lt/le/gt/ge`, `isNull/isNotNull`, + `and/or/not` with typed value overloads for Number, String, Boolean, LocalDate, LocalDateTime, + Instant, LocalTime and byte[]. Convert dates, timestamp/time units, UTC adjustment, decimals + (including integer and fixed binary physical forms) using the actual file schema. Reject + overflow, lossy rescaling, incompatible local/instant comparisons and unsupported repeated-field + predicates. Implement exact logical residual evaluation when safe pushdown is unavailable; do + not incorrectly compare signed decimal bytes as generic lexicographic binary. Partition + pushdown/residual expressions safely, especially under OR/NOT. Document null/NaN semantics and + test them against full evaluation. + +- 3.6 [x] Remap `matrix.columnTypes` by original field name for projected/reordered columns. + Restore Matrix index metadata only when all indexed columns survive; otherwise omit that Matrix + index. Empty matches must still retain the selected schema and types. + +- 3.7 [x] Test result equivalence with index filtering on/off and indexes absent. Cover AND/OR, + null predicates, unordered data, truncated string bounds, boundary values and filter-only + columns. Treat unavailable indexes as a normal scan fallback; propagate genuine corruption + rather than silently returning partial results. + +- 3.8 [x] Prove page skipping using a counting seekable input and controlled multi-page fixture. + Disable row-group/dictionary/bloom pruning in this focused test so an entire row-group skip + cannot masquerade as page-index use. Require identical results and fewer data-page bytes read; + avoid timing assertions and exact read-call counts. + +- 3.9 [x] Add `limit(long)` (default unlimited/null, nonnegative; zero returns typed empty + output), `rowGroups(List)` (default all, zero-based, unique IDs in caller-specified + order), and `batches(...)` returning a Matrix-owned closeable iterator of Matrix values, + normally one batch per selected row group. Limit counts matching rows after filtering and + applies across batches. Footer IDs in range remain valid when statistics, dictionary or bloom + pruning excludes them: contribute no rows and skip the empty batch. Only out-of-range or + duplicate IDs are rejected. Preserve caller-selected order and test valid pruned IDs. Close + resources on exhaustion, explicit close and failure; document try-with-resources/withCloseable. + Iterator state stays on one input and does not prefetch unbounded groups. One row group can + exceed heap, so also support `maxBatchRows` (positive, default null) to split it; document that + byte-array cells/one huge row still constrain memory. Skip empty filtered batches and retain + schema for empty materialized results. Test early stop and cleanup. + +- 3.10 [x] Measure 3.2/3.3 against 1.6 before removing the old path. Initial acceptance target: at + least 20% lower median full-scan elapsed time on wide text and 25% lower peak live heap on + categorical data, with no greater than 5% median regression on mixed/nested data. Record + repeated-run variance; revise targets explicitly if evidence shows they are unrealistic. Timing + thresholds belong in benchmark reports, not flaky CI assertions. + +### 4. Add selective HTTP access (P2, depends on 3) + +- 4.1 [x] Implement a seekable HTTP `InputFile` / `SeekableInputStream` with bounded range + caching. Discover stable length and verify actual `206`/`Content-Range` behavior; fetch + footer/index/data ranges on demand. Reuse the same remote-object identity for metadata and rows, + detecting changes via validators where available. + +- 4.2 [x] Preserve ordinary URL reads through fallback to the existing temporary-file path when a + server lacks range support. Proposed `remoteReadMode` enum/string default `DOWNLOAD` preserves + current behavior; `AUTO` attempts ranges then downloads safely, and `RANGE` requires verified + range support. Add positive `connectTimeoutMillis` and `readTimeoutMillis`, proposed defaults + 10000/60000, plus positive `rangeCacheBytes`, proposed default 8388608. Propagate these through + builder/options/SPI. + +- 4.3 [x] Test redirects typical of Hub resolve URLs, servers ignoring Range, bad range responses, + EOF, retries with bounded attempts, resource cleanup and object changes. Do not forward + authorization across unrelated redirect origins. Keep credentials out of logs. No new Hub + authentication or Xet-client integration is required for public URLs. + +- 4.4 [x] Use a local range-capable HTTP server and selective projection/predicate fixture to + prove fewer transferred bytes than a full download. Count footer/index overhead too. Document + that this is HTTP range access, not Xet deduplication or a streaming Matrix. + +### 5. Confirm writing behavior; defer native CDC to a separate decision (P1/P3) + +- 5.1 [x] Based on 1.4, document existing page-index output. If an eligible-column omission is + reproducible, fix shared writer configuration across file/memory/stream paths; extract common + configuration rather than copying more builder chains. Add page-size tuning through the + explicitly planned options in 5.4; benchmarks inform guidance, not whether the options are + implemented. + +- 5.2 [x] Keep native CDC writing deferred unless a supported parquet-java release exposes it or a + separately scoped upstream implementation is accepted. No private upstream patch, mandatory + Python bridge, or misleading “optimized” switch in this enhancement. Page-size/row-group tuning + alone is not CDC. + +- 5.3 [x] For users needing CDC output now, document the external PyArrow rewrite below, verify + values and custom metadata survive, and state the extra I/O and Python requirement. A future + native CDC proposal must cover per-column logical hashing, nested levels, dictionary effects, + deterministic output and measured reuse after inserts/deletes. + +- 5.4 [x] Expose the safe tuning subset below through typed options, builder, SPI and shared + writer configuration. Use a Matrix-owned writer-version enum/string. Validate positive + sizes/NDVs, primitive column paths and names against the resolved schema; reject unknown map + keys/columns and invalid versions. Keep current defaults explicitly. + + | Option | Type | Proposed default | Meaning | + |---|---|---|---| + | `rowGroupSize` | long | 134217728 | Target row-group bytes | + | `pageSize` | int | 1048576 | Target encoded data-page bytes | + | `dictionaryEncoding` | boolean | true | Enable dictionary encoding with upstream fallback | + | `writerVersion` | Matrix enum/string | V1 | Select V1/V2 writer encodings and page layout | + | `columnIndexTruncateLength` | int | 64 | Binary bounds truncation for page indexes | + | `statisticsTruncateLength` | int | 2147483647 | Binary bounds truncation for row-group statistics | + | `bloomFilterEnabled` | Map | empty | Per-column overrides, globally disabled | + | `bloomFilterNDV` | Map | empty | Per-column expected distinct values; omission retains upstream sizing | + | `sizeStatisticsEnabled` | boolean | true | Collect upstream size statistics | + + V2 selects a different encoder factory as well as page layout; do not describe this as just a + page-header toggle. Keep V1 default and document that older consumers may not support all V2 + encodings; name specific consumers only after compatibility testing. Describe target-size versus + hard-limit semantics, NDV memory tradeoffs and disabled-column interactions. Test physical + metadata/page versions and logical equivalence across every output path. Add indexed equality + tests proving bloom-filter pruning independently from page/dictionary/row-group-statistics + pruning. Match any changed defaults to versioned upstream constants before implementation. + +- 5.5 [x] Track native CDC in P3: investigate [parquet-java + #3818](https://github.com/apache/parquet-java/pull/3818), referenced by + [Arrow #51684](https://github.com/apache/arrow/issues/51684) as a Java CDC implementation. The + Java PR remains open and unmerged as verified on 2026-10-17; 1.18.1 exposes no supported + CDC builder API. Recheck before choosing a future dependency. Link the applicable issue/PR and supported release here, + or draft a concrete upstream issue covering hooks/page boundaries, nested hashing and + interoperability. Ask for authorization before posting externally; a reviewable issue draft is + sufficient until then. Do not silently leave deferral untracked. + +### 6. Documentation, measurements and completion gates + +- 6.1 [x] Update module README, `docs/cookbook/matrix-parquet.md`, + `docs/tutorial/11-matrix-parquet.md` and release notes. Distinguish tested full-file + compatibility, selective local reading, selective HTTP reading and CDC writing. Include complete + examples, every new parameter/default, supported types and fallback behavior. Do not advertise + arbitrary ordinal random access from a filter API; `withFileRange` is a byte range, not a row + interval. Ordinal seeking is a separate feature. + +- 6.2 [x] Record a reproducible benchmark for full scan, narrow projection and selective + predicates on ordinary/indexed/CDC+indexed files: rows, file/page/row-group sizes, compression, + returned rows, bytes read/transferred, wall time and peak memory. Keep timing benchmarks outside + normal correctness tests; compare full-scan results to 1.6 and the measured targets in 3.10. + +- 6.3 [x] For each implementation phase run, in order: `./gradlew :matrix-parquet:codenarcMain`, + `./gradlew :matrix-parquet:spotlessCheck`, `./gradlew :matrix-parquet:test`. Run `./gradlew + test` before completing implementation. Record successful commands and outcomes here or in the + PR before checking tasks off. Include `codenarcTest` for changed tests. + +### 7. Sharded datasets and bounded parallel decoding (P2) + +- 7.1 [x] Add `read(List)`, directory input and explicit `readGlob(String)` using caller + order for lists and sorted normalized-path order for directories/globs; reject duplicate + normalized paths. No implicit glob interpretation in ordinary file strings. Directory reads + select direct `.parquet` children by default, not recursive trees. Reject empty selections with + a clear error. Check all schemas before collecting results: strict logical types, field + names/order, decimal precision/scale and timestamp units/UTC flags by default. Do not silently + coerce incompatible shards. Test Matrix metadata policy separately from physical schema + compatibility. + +- 7.2 [x] Add `sourceFileColumn` (String, default null; reject collisions) containing each shard's + normalized absolute local path. Apply projection/predicates independently per file schema and a + single matching-row limit across the dataset. Define dataset row-group selectors as explicit + per-file selections rather than ambiguous global group numbers. Extend closeable batches to + shards while preserving deterministic file/group order. + +- 7.3 [x] Add `parallelism` (positive int, default 1) and bounded prefetch (default one work item + per worker). Decode independent files/groups on separate readers; never share a seekable stream, + converter or mutable dictionary. Reuse each file's already-parsed footer through + `ParquetFileReader.open(InputFile, ParquetMetadata, ParquetReadOptions, SeekableInputStream)` + with a separate stream per worker. Verify the source identity/length matches the footer snapshot + and close streams on constructor/open failures. Test one footer parse per file rather than per + worker; do not share mutable pruning state between workers. Preserve order, cap queued results, + cancel remaining work on limit/close/failure and propagate errors with source context. Measure + throughput/heap against sequential decoding and test resource cleanup and deterministic results; + avoid decoding whole datasets before returning the first batch. + +### 8. Optional Hub URL resolution (P3) + +- 8.1 [x] Specify and test `hf://datasets//@/` resolution to an HTTPS + `/datasets///resolve//` URL; revision defaults to `main`. Match + huggingface_hub parsing/quoting, including percent-encoded revision slashes + (`refs%2Fconvert%2Fparquet`) and supported special unescaped refs. Pin compatibility to + huggingface_hub **v2.2.0** `parse_hf_uri` and URL quoting, after comparing v1.0.0 `resolve_path`, + `unquote`, special-ref regex and `safe_revision`. The new convert-ref regex allows dots/dashes; + deep-path @ is literal. Record differences and regenerate offline differential reference cases + with the adopted tagged client. Preserve unescaped refs/convert/parquet and refs/pr/N; percent-encode + other slash-containing revisions according to the selected client. Add differential URL tests + for @, nested paths, encoded and special refs, Unicode and avoiding double decoding/encoding. + [Client implementation](https://github.com/huggingface/huggingface_hub/blob/v2.2.0/src/huggingface_hub/hf_file_system.py). + Reuse phase 4 transport without adding Xet claims. Optional HF_TOKEN is sent only to approved + `https://huggingface.co` origins, never signed CDN redirect origins or logs; test redirect + behavior with mocked origins. Verify current Hub URL/auth documentation before implementation. + Existing public HTTPS reads need no token. + +Execution order: **P0: 0–2; P1: 3 and 5.1/5.4; P2: 7 and 4; P3: 8 and 5.5**. +Documentation and completion checks in 6 apply to every delivery. Phase 5.2/5.3 explains +CDC deferral and the external rewrite. Native CDC writing is not a prerequisite for reading +optimized downloads correctly. The execution record below describes the delivered implementation. + +## Usage examples and fixture reproduction + +API for a downloaded optimized file (compatibility verified by phase 1): + +```groovy +import se.alipsa.matrix.parquet.MatrixParquetReader + +def data = MatrixParquetReader.read(new File('downloaded-optimized.parquet')) +``` + +Phase 3 API; values convert through the file schema: + +```groovy +import se.alipsa.matrix.parquet.Predicates +import se.alipsa.matrix.parquet.MatrixParquetReader + +def subset = MatrixParquetReader.builder() + .columns(['id', 'text']) + .filter(Predicates.ge('id', 1000L)) + .limit(100) + .useColumnIndexFilter(true) + .read(new File('downloaded-optimized.parquet')) +``` + +Additional P1/P2 usage: + +```groovy +import java.time.LocalDate +import se.alipsa.matrix.parquet.MatrixParquetReader +import se.alipsa.matrix.parquet.Predicates +import se.alipsa.matrix.parquet.MatrixParquetWriter + +// Logical DATE comparisons require no epoch-day conversion in application code. +def preview = MatrixParquetReader.builder() + .filter(Predicates.ge('date', LocalDate.of(2026, 1, 1))) + .limit(20) + .read(new File('events.parquet')) + +// Close explicitly even if consumption stops early; limit counts matches globally. +MatrixParquetReader.builder().limit(1000).maxBatchRows(250) + .batches(new File('events.parquet')).withCloseable { batches -> + while (batches.hasNext()) { + def batch = batches.next() + // Process this Matrix before requesting the next bounded batch. + } + } + +MatrixParquetWriter.builder(preview) + .rowGroupSize(16L * 1024 * 1024) + .pageSize(64 * 1024) + .dictionaryEncoding(true) + .writerVersion('V2') + .write(new File('preview.parquet')) + +// Shard order follows this supplied list; directory/glob APIs sort paths. +def shards = MatrixParquetReader.builder().sourceFileColumn('_source').parallelism(2) + .read([new File('train-00000.parquet'), new File('train-00001.parquet')]) +``` + +For explicit lists, preserve caller order (and reject duplicate normalized paths); for +folder/glob selections use sorted normalized paths. Carry this distinction into 7.1. + +Minimal end-to-end external rewrite for CDC output. Use a dedicated environment and pin +the generator version in the fixture manifest; 21.0.0 is a suitable minimum baseline: + +```bash +python3 -m venv /tmp/matrix-parquet-fixtures +/tmp/matrix-parquet-fixtures/bin/python -m pip install pyarrow==21.0.0 +/tmp/matrix-parquet-fixtures/bin/python - <<'PY' +import pyarrow.parquet as pq + +# input.parquet is an existing matrix-parquet output file. +table = pq.read_table('input.parquet') +pq.write_table( + table, 'optimized.parquet', compression='snappy', + use_content_defined_chunking=True, write_page_index=True, +) +assert pq.read_table('optimized.parquet').equals(table, check_metadata=True) +PY +``` + +Here `table` is the full Arrow table, the second argument is the output path, +`compression` explicitly chooses SNAPPY (also PyArrow's default), CDC defaults to false +unless enabled, and page-index writing defaults to false unless enabled. Other settings +use the pinned PyArrow version's defaults. This example loads the table into memory; +the eventual fixture generator must explicitly set row-group/page/encoding options and +emit all four CDC/index combinations plus its manifest. It must not require Hub uploads. + + +## Implementation and verification record — 2026-10-17 + +All numbered deliverables above are implemented and verified in this branch. Native CDC writing +is intentionally deferred as specified by 5.2; its upstream tracking and external rewrite are +complete. The module remains on parquet-java 1.18.1, Hadoop 3.5.0 and JDK 21. + +### Final ordered gates + +These successful commands verify the delivered tree, including each separate 2.0a–2.0e task: + +```bash +./gradlew :matrix-parquet:codenarcMain :matrix-parquet:codenarcTest --console=plain +./gradlew :matrix-parquet:spotlessCheck --console=plain +PARQUET_PYTHON=/tmp/matrix-parquet-fixtures/bin/python \ + ./gradlew :matrix-parquet:test :matrix-parquet:compileConsumerApi -PrunSlowTests=true --console=plain +./gradlew test -Pheadless=true --console=plain +``` + +CodeNarc and Spotless passed. The Parquet suite reported **122 tests, 117 passed, zero failures, +five skipped**. The skips are opt-in benchmarks and original-writer fixture capture; default-size +CDC was generated and tested. The Java consumer compiles without parquet-java/Hadoop on its +compile classpath. The full repository regression task passed; unaffected tests may be up-to-date. + +| Task | Implementation and independently asserted coverage | +|---|---| +| 0.1–1.5 | 0.8.0 migration table; original 0.7.x strings/temporal/decimal fixtures; eight Arrow layout cases and manifest; physical indexes/pages; all primary entry points | +| 2.0 | Recursive STRING annotations, version marker, automatic legacy nested text compatibility and binaryAsString options | +| 2.0a | Raw binary writes and exact reads through bytes/files/streams/SPI; fixed metadata allowlist; unknown/incompatible metadata fallback; explicit legacy `[B` rejection | +| 2.0b | Byte/Short/LocalTime/Instant/UUID/Character/Enum policies; precision/overflow checks; contextual unsupported-type rejection | +| 2.0c | Shared ColumnTypePolicy descriptor; complete scans, abstract/interface inference, concrete assignability, all-null rules and incompatible Date subclass rejection in either order | +| 2.0d | Exact inferred/declared/nested BigDecimal DECIMAL routing; per-field/global precedence, disabled-inference errors, high precision, negative scale and all-null cases | +| 2.0e | Allowlist coverage for all emitted canonical types; every supported policy through bytes, streams and SPI files; validation before output mutation and migration notes | +| 2.1–2.5 | External raw media bytes, integer decimals, naive/adjusted timestamps at every unit, UUID and explicit FLOAT16/INT96 errors, legacy LIST variants | +| 3.1–3.9 | InputFile/ParquetFileReader core, column converters, accept/skip rollback, dictionary reuse, logical predicates, projection/index remapping, physical pruning, limits and closeable bounded batches | +| 4.1–4.4 | Range input, fallback, cache/timeouts, bounded redirects/retries, origin-scoped tokens, malformed/truncated ranges, object changes and fewer transferred bytes | +| 5.1–5.5 | Physical output indexes across file/memory/stream paths, shared tuning, bloom-only pruning, metadata-preserving Arrow CDC rewrite and upstream CDC tracking | +| 6.1, 6.3 | Complete README/defaults/examples, tutorial/cookbook/release updates; ordered gates and full repository suite | +| 1.6, 3.10, 6.2, 7.3 | Original wide baseline, million-row Group/columnar comparisons, sampled live heap, three Arrow layouts with full/projection/filter modes and sequential/parallel dataset measurements | +| 7.1–7.3 | Ordered list/directory/glob shards, strict schemas/metadata, source column, per-file groups, reused footers, bounded independent workers, limit and early-close cleanup | +| 8.1 | Tagged huggingface_hub 2.2.0 offline differential cases after v1.0.0 comparison; revision/path quoting and redirect credential tests | + +### Reproduction and additional successful checks + +```bash +/tmp/matrix-parquet-fixtures/bin/python matrix-parquet/src/test/python/generate_optimized.py +/tmp/matrix-hf-fixtures/bin/python matrix-parquet/src/test/python/generate_hub_urls.py +/tmp/matrix-parquet-fixtures/bin/python matrix-parquet/src/test/python/verify_matrix_output.py +./gradlew :matrix-parquet:test --tests ParquetInteropTest --console=plain +./gradlew :matrix-parquet:codecWithoutHadoop --console=plain +groovy scripts/release-all/test-workflow.groovy . +``` + +Fixture regeneration is byte-for-byte deterministic on the pinned environments. Checked-in +Parquet binaries total **737,019 bytes**, within the 4 MiB budget and 1 MiB individual cap. +Python environments use pyarrow==21.0.0 and huggingface_hub==2.2.0 respectively. Python verification +confirmed SNAPPY/ZSTD annotated strings, binary payloads and exact decimals; CDC rewrites retain +values and Matrix metadata and are reread by Java. CI runs these checks in +.github/workflows/release-all-tools.yml. Fixture provenance and client-version differences are +recorded in src/test/resources/optimized/README.md. + +Original-writer capture was run successfully with: + +```bash +./gradlew :matrix-parquet:test --tests ParquetLegacyCaptureTest \ + -I /tmp/parquet-legacy.init.gradle --no-configuration-cache --console=plain +``` + +That init script sets test system property parquet.captureLegacy=true. The capture compiles the +writer/value helper at b2095159664b34ce426685c387adf0bccdb33e01 together in an isolated loader. +The byte[] hypothesis was corrected by this end-to-end experiment: this pinned writer throws +InvalidSchemaException and cannot produce a valid byte[] fixture. The checked-in `[B` text footer +is explicitly synthetic; no original bytes are claimed recoverable. Date/Timestamp corruption is +preserved in an actual original-writer fixture and new mixed policies are rejected before output. + +The codec-exclusion spike showed that even UNCOMPRESSED writer initialization loads Hadoop's +ZlibCompressor.CompressionLevel. Local filesystem/configuration setup is removed from the new +read/write core, but runtime Hadoop dependencies remain until upstream codec linkage permits +removal. No JDK target increase or dependency upgrade is claimed. + +### Measurements + +See [the benchmark report](../benchmarks/optimized-parquet.md) for hardware/JVM/heap/GC, input +generation, warmup/variance, allocations, sampled live-heap definitions, exact byte counts and +reproduction commands. The original wide baseline was captured before production edits; the +three-workload differential comparison retains the original Group loop with shared corrected +schema conversions. Inputs are generated outside Git, and no claim of an OS-cold cache is made. + +The million-row comparison measured 89.5% lower wide median time, 43.8% lower categorical sampled +live heap and 88.2% lower mixed/nested time, meeting the initial targets with the documented +sampling method. Identical Arrow layouts exercise full scans, ID projection and 100-row selection; +page indexes reduced selective reads from 1,279,020 to 552,869 bytes (542,383 with CDC). An isolated +page-index test reads 2,914 versus 82,700 bytes with all other pruning disabled. Independent bloom +pruning reads 16,737 versus 40,373 bytes. HTTP ranges transfer 88,476 bytes of an approximately 906,000-byte object, +including footer overhead. Four-worker shard decoding measured a 242 ms warm median versus 341 ms +sequentially. These are local observations, not timing assertions or general speed guarantees. + +Benchmark commands (init-file setup is fully documented in the report): + +```bash +./gradlew :matrix-parquet:test --tests '*ParquetReadBenchmarkTest' \ + -I /tmp/parquet-baseline.init.gradle --no-configuration-cache --console=plain +./gradlew :matrix-parquet:test --tests '*ParquetDatasetBenchmarkTest' \ + -I /tmp/parquet-dataset.init.gradle --no-configuration-cache --console=plain +/tmp/matrix-parquet-fixtures/bin/python matrix-parquet/src/test/python/generate_benchmarks.py +./gradlew :matrix-parquet:test --tests '*ParquetLayoutBenchmarkTest' \ + -I /tmp/parquet-layout.init.gradle --no-configuration-cache --console=plain +``` + + +The subsequent PR review fixes and their verification are recorded in +[PR #481 review fixes](optimized-parquet-review.md). diff --git a/matrix-parquet/req/optimized-parquet-review.md b/matrix-parquet/req/optimized-parquet-review.md new file mode 100644 index 000000000..a0f21a03e --- /dev/null +++ b/matrix-parquet/req/optimized-parquet-review.md @@ -0,0 +1,122 @@ +# 9. PR #481 review fixes + +The numbered tasks correspond to the thirteen findings in the review. The fixes were developed +on `feature/optimized-parquet-review`, based on the original PR commit eee2b16e. + +- 9.1 [x] Handle empty dictionaries without a descending Groovy range. Add the exact three-row + all-null PyArrow string fixture, generator provenance and a read regression. +- 9.2 [x] Decode JSON and ENUM annotations as UTF-8 String, including dictionary conversion; + retain BSON bytes. Test PyArrow JSON and Java ENUM/BSON with binary overrides and update the + migration table. +- 9.3 [x] Separate the 8 MiB request cap from the configured cache ceiling. Bulk reads use the + requested span; reuse validated redirect targets, refresh rejected signed targets through the + original URL, and leave fully consumed successful range connections eligible for keep-alive. + Measure actual request counts with a local redirect/range server. +- 9.4 [x] Route static URL/URL-string overloads through the builder transport, including named + and zoned hf:// overloads. Test bounded timeouts, downgrade rejection and Hub parsing. +- 9.5 [x] Write local files to temporary siblings and atomically replace the destination only + after successful close. Preserve existing POSIX permissions and clean temporary files on + failure. Test rounding and precision failures after a valid row and compare original bytes. +- 9.6 [x] Support sourceFileColumn for local single-file reads and batches. Restore common + compatible indices for both collected and batched dataset reads, including truncated batches; + omit indices when projection removes their columns or shards disagree on index metadata. +- 9.7 [x] Resolve each top-level column's validated type policy once and carry it in WriteSchema + through decimal/schema construction and metadata emission. Remove obsolete duplicate inference + helpers and report full nested logical paths; test nested error context. +- 9.8 [x] Treat null option-map values as absent consistently, preserving Boolean/default/map + values and nullable AUTO semantics. Reject null bloom entries with contextual validation. +- 9.9 [x] Create dataset worker threads through a daemon ThreadFactory. Assert their daemon + status while an iterator remains open; explicit close is still required for prompt cleanup. +- 9.10 [x] Route named/zoned File/Path/string overloads through the shared directory-capable + builder and test directory results, names, time-zone values and index restoration. +- 9.11 [x] Reject literal dot/parent path segments and decoded revision dot/parent segments; + retain the pinned client URL cases. Test nested paths and encoded revision traversal. +- 9.12 [x] Reject unrecoverable legacy [B metadata only for required output/filter columns. + Test successful projection around the damaged column and rejection when a filter needs it. +- 9.13 [x] Correct the reader's memory/batch/cache comments and writer option documentation, + including literal V1 defaults; update source/index/transport behavior in the module guide. + +## Verification + +The required ordered module gates passed: + +```bash +./gradlew :matrix-parquet:codenarcMain :matrix-parquet:codenarcTest --console=plain +./gradlew :matrix-parquet:spotlessCheck --console=plain +PARQUET_PYTHON=/tmp/matrix-parquet-fixtures/bin/python \ + ./gradlew :matrix-parquet:test :matrix-parquet:compileConsumerApi -PrunSlowTests=true --console=plain +./gradlew test -Pheadless=true --console=plain +``` + +The Parquet suite reported **133 tests, 128 passed, zero failed, five intentional opt-in +benchmark/capture skips**. Default-size CDC generation/read and the downstream consumer compile +probe passed. The full repository task passed; unaffected tasks may be up-to-date. + +Additional successful checks: + +```bash +/tmp/matrix-parquet-fixtures/bin/python matrix-parquet/src/test/python/generate_optimized.py +/tmp/matrix-parquet-fixtures/bin/python matrix-parquet/src/test/python/verify_matrix_output.py +``` + +Fixture regeneration is deterministic. Checked-in Parquet binaries total **738,113 bytes**, +below the 4 MiB aggregate and 1 MiB individual limits. Arrow verifies SNAPPY/ZSTD strings, +binary payloads, exact decimals and metadata-preserving CDC rewrites. + +HTTP request-count assertions show one 1 MiB data request after probing, one redirect resolution +per input, and no extra request when the default cache serves the repeated block. A 1 KiB cache +still permits the same single 1 MiB bulk request. The test checks byte-array and ByteBuffer +readFully paths. Successful bodies remain eligible for JDK keep-alive pooling; socket-level +connection reuse and live Hub latency were not measured. Existing selective-transfer, page-index +and bloom-filter byte-reduction tests also pass. + +## 10. Follow-up output and range-buffer review + +- 10.1 [x] Create temporary siblings with ordinary file creation permissions, so new output + follows the process umask. Compare new Parquet permissions with an ordinary sibling file on + POSIX, while retaining the existing-destination permission regression. +- 10.2 [x] Resolve existing output paths before creating the sibling and replacing the target. + Test that relative symbolic links survive both failed and successful writes, and that the + resolved file retains its contents on failure and receives the new rows on success. +- 10.3 [x] Document the same-directory atomic replacement requirement and + `AtomicMoveNotSupportedException` in the release notes and module guide. Keep safe failure + rather than introduce a replacement fallback with weaker failure guarantees. +- 10.4 [x] Read directly into array-backed ByteBuffers using their array offset and current + position. Retain the bounded scratch buffer for direct buffers. Exercise sliced buffers with + nonzero positions, sentinel bytes outside the slice, direct buffers, EOF and empty reads in + the existing HTTP request-count regression. + +Follow-up verification passed, in the required order: + +```bash +./gradlew :matrix-parquet:codenarcMain :matrix-parquet:codenarcTest :matrix-parquet:spotlessCheck --console=plain +PARQUET_PYTHON=/tmp/matrix-parquet-fixtures/bin/python \ + ./gradlew :matrix-parquet:test :matrix-parquet:compileConsumerApi -PrunSlowTests=true --console=plain +./gradlew test -Pheadless=true --console=plain +``` + +The Parquet suite reported **135 tests, 130 passed, zero failed, five intentional skips**, +including successful POSIX permission/symlink tests, Python interoperability and consumer API +compilation. The full repository task passed; unaffected tasks may be up-to-date. + +## 11. Dangling symlinks and interrupted-write documentation + +- 11.1 [x] Resolve symbolic links even when their destination does not exist, including relative + and absolute targets and link chains. Reject cycles before creating temporary output. Test + successful target creation and failed writes preserving dangling links and absent targets. +- 11.2 [x] Explain that abrupt JVM termination may leave `.matrix-parquet-.tmp` siblings + beside the resolved output and that unused leftovers can be removed. + +Verification passed in order: + +```bash +./gradlew :matrix-parquet:codenarcMain :matrix-parquet:codenarcTest --console=plain +./gradlew :matrix-parquet:spotlessCheck --console=plain +PARQUET_PYTHON=/tmp/matrix-parquet-fixtures/bin/python \ + ./gradlew :matrix-parquet:test :matrix-parquet:compileConsumerApi -PrunSlowTests=true --console=plain +./gradlew test -Pheadless=true --console=plain +``` + +The Parquet suite reported **137 tests, 132 passed, zero failed, five intentional skips**. +The POSIX dangling-link and cycle tests ran successfully, as did Python interoperability, +consumer API compilation and the full repository suite; unaffected tasks may be up-to-date. diff --git a/matrix-parquet/src/consumerTest/java/ParquetConsumer.java b/matrix-parquet/src/consumerTest/java/ParquetConsumer.java new file mode 100644 index 000000000..23e71f202 --- /dev/null +++ b/matrix-parquet/src/consumerTest/java/ParquetConsumer.java @@ -0,0 +1,20 @@ +import java.io.File; +import java.util.List; +import se.alipsa.matrix.core.Matrix; +import se.alipsa.matrix.parquet.MatrixParquetReader; +import se.alipsa.matrix.parquet.MatrixParquetWriter; +import se.alipsa.matrix.parquet.Predicates; + +/** Compile-only check: consumers need no parquet-java or Hadoop on their compile classpath. */ +public final class ParquetConsumer { + /** Selects an ID preview without upstream predicate types. */ + public static Matrix select(File file) { + return MatrixParquetReader.builder().columns(List.of("id")) + .filter(Predicates.ge("id", 1000L)).limit(10).read(file); + } + + /** Writes using Matrix-owned tuning methods. */ + public static void save(Matrix matrix, File file) { + MatrixParquetWriter.builder(matrix).pageSize(4096).writerVersion("V2").write(file); + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnMaterializer.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnMaterializer.groovy new file mode 100644 index 000000000..6e92eb0ca --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnMaterializer.groovy @@ -0,0 +1,175 @@ +package se.alipsa.matrix.parquet + +import groovy.transform.PackageScope + +import org.apache.parquet.column.Dictionary +import org.apache.parquet.io.api.Binary +import org.apache.parquet.io.api.Converter +import org.apache.parquet.io.api.GroupConverter +import org.apache.parquet.io.api.PrimitiveConverter +import org.apache.parquet.io.api.RecordMaterializer +import org.apache.parquet.schema.GroupType +import org.apache.parquet.schema.LogicalTypeAnnotation +import org.apache.parquet.schema.MessageType +import org.apache.parquet.schema.PrimitiveType +import org.apache.parquet.schema.Type + +/** Column-oriented materialization; record acceptance and rollback remain in upstream hooks. */ +@PackageScope +final class ColumnMaterializer extends RecordMaterializer { + + final List columns + final Map current = [:] + private final GroupConverter root + private final int[] sizes + private final ParquetPredicate residual + + ColumnMaterializer(MessageType schema, List types, boolean binaryAsString, ParquetPredicate residual) { + columns = schema.fields.collect { [] } + sizes = new int[columns.size()] + this.residual = residual + Converter[] children = new Converter[schema.fieldCount] + schema.fields.eachWithIndex { Type field, int index -> + Closure logicalSink = residual != null && residual.columns().contains(field.name) ? { Object value -> current[field.name] = value } : null + children[index] = converter(field, types[index], binaryAsString, + { Object value -> columns[index].add(value) }, logicalSink) + } + root = new GroupConverter() { + @Override + Converter getConverter(int index) { children[index] } + + @Override + void start() { + current.clear() + columns.eachWithIndex { List column, int index -> sizes[index] = column.size() } + } + + @Override + void end() { + columns.eachWithIndex { List column, int index -> + if (column.size() == sizes[index]) { column.add(null) } + } + } + } + } + + @Override + GroupConverter getRootConverter() { root } + + @Override + Boolean getCurrentRecord() { + residual == null || residual.test(current) ? Boolean.TRUE : null + } + + @Override + void skipCurrentRecord() { + columns.eachWithIndex { List column, int index -> + if (column.size() > sizes[index]) { column.subList(sizes[index], column.size()).clear() } + } + } + + private static Converter converter(Type type, Class expected, boolean binaryAsString, Closure sink, Closure logicalSink = null) { + if (type.primitive) { + return new ValueConverter(type.asPrimitiveType(), expected, binaryAsString, sink, logicalSink) + } + GroupType group = type.asGroupType() + Map values = [:] + Converter[] children = new Converter[group.fieldCount] + group.fields.eachWithIndex { Type child, int index -> + children[index] = converter(child, null, binaryAsString) { Object value -> + if (child.repetition == Type.Repetition.REPEATED) { + ((List) values.computeIfAbsent(child.name) { [] }).add(value) + } else { + values[child.name] = value + } + } + } + new GroupConverter() { + @Override + Converter getConverter(int index) { children[index] } + + @Override + void start() { values.clear() } + + @Override + void end() { + def logical = group.logicalTypeAnnotation + if (logical instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) { + Type repeated = group.getType(0) + List entries = (List) (values[repeated.name] ?: []) + boolean wrapped = !repeated.primitive && repeated.asGroupType().fieldCount == 1 && + !(repeated.name in ['array', "${group.name}_tuple".toString()]) + sink.call(wrapped ? entries.collect { entry -> ((Map) entry)[repeated.asGroupType().getType(0).name] } : entries) + } else if (logical instanceof LogicalTypeAnnotation.MapLogicalTypeAnnotation) { + Map result = [:] + ((List) (values[group.getType(0).name] ?: [])).each { entry -> Map pair = (Map) entry; result[pair.key] = pair.value } + sink.call(result) + } else { + Map result = [:] + group.fields.each { Type child -> result[child.name] = values[child.name] } + sink.call(result) + } + } + } + } + + private static class ValueConverter extends PrimitiveConverter { + private final PrimitiveType field + private final Class expected + private final boolean binaryAsString + private final Closure sink + private final Closure logicalSink + private Object[] dictionary + private Object[] logicalDictionary + + ValueConverter(PrimitiveType field, Class expected, boolean binaryAsString, Closure sink, Closure logicalSink = null) { + this.field = field + this.expected = expected + this.binaryAsString = binaryAsString + this.sink = sink + this.logicalSink = logicalSink + } + + private void accept(Object value) { + sink.call(MatrixParquetReader.decodePrimitive(value, field, expected, binaryAsString)) + if (logicalSink != null) { logicalSink.call(logicalValue(value)) } + } + + private Object logicalValue(Object value) { + Class logicalExpected = field.logicalTypeAnnotation instanceof LogicalTypeAnnotation.TimeLogicalTypeAnnotation ? java.time.LocalTime : + field.logicalTypeAnnotation instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation && + ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) field.logicalTypeAnnotation).isAdjustedToUTC() ? java.time.Instant : + field.logicalTypeAnnotation == null && field.primitiveTypeName == PrimitiveType.PrimitiveTypeName.BINARY ? byte[] : null + MatrixParquetReader.decodePrimitive(value, field, logicalExpected, false) + } + + @Override + void addInt(int value) { accept(value) } + @Override + void addLong(long value) { accept(value) } + @Override + void addFloat(float value) { accept(value) } + @Override + void addDouble(double value) { accept(value) } + @Override + void addBoolean(boolean value) { accept(value) } + @Override + void addBinary(Binary value) { accept(value) } + @Override + boolean hasDictionarySupport() { + MatrixParquetReader.isTextLogicalType(field.logicalTypeAnnotation) || expected == String + } + @Override + void setDictionary(Dictionary source) { + dictionary = new Object[source.maxId + 1] + if (logicalSink != null) { logicalDictionary = new Object[source.maxId + 1] } + for (int id = 0; id <= source.maxId; id++) { + Binary value = source.decodeToBinary(id) + dictionary[id] = MatrixParquetReader.decodePrimitive(value, field, expected, binaryAsString) + if (logicalSink != null) { logicalDictionary[id] = logicalValue(value) } + } + } + @Override + void addValueFromDictionary(int id) { sink.call(dictionary[id]); logicalSink?.call(logicalDictionary[id]) } + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnTypePolicy.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnTypePolicy.groovy new file mode 100644 index 000000000..a3549f13c --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnTypePolicy.groovy @@ -0,0 +1,105 @@ +package se.alipsa.matrix.parquet + +import groovy.transform.PackageScope + +import java.lang.reflect.Modifier +import java.sql.Time +import java.sql.Timestamp +import java.time.Instant +import java.time.LocalDate +import java.time.LocalDateTime +import java.time.LocalTime + +/** Validated encoding and metadata types shared by schema construction and metadata emission. */ +@PackageScope +final class ColumnTypePolicy { + + private static final int NANOS_PER_MICRO = 1000 + static final List SUPPORTED = [String, Integer, Long, Float, Double, Boolean, Byte, Short, + BigInteger, BigDecimal, Character, byte[], LocalDate, LocalDateTime, LocalTime, Instant, + UUID, Date, java.sql.Date, Time, Timestamp, List, Map] + static final Map METADATA_TYPES = SUPPORTED.collectEntries { Class type -> [(type.name): type] } + + final Class encoding + final Class metadata + + private ColumnTypePolicy(Class encoding, Class metadata) { + this.encoding = encoding + this.metadata = metadata + } + + static ColumnTypePolicy resolve(String name, List values, Class declaration) { + Class declared = box(declaration ?: Object) + boolean infer = declared == Object || declared.interface || Modifier.isAbstract(declared.modifiers) + Class policy = null + values.eachWithIndex { value, int row -> + if (value == null) { + return + } + if (!declared.isInstance(value)) { + throw new IllegalArgumentException("Column '$name', row $row: declared ${declared.name}, got ${value.getClass().name}") + } + if (value instanceof Instant) { + if (value.nano % NANOS_PER_MICRO != 0) { throw new IllegalArgumentException("Column '$name', row $row: Instant loses sub-microsecond precision") } + try { Math.addExact(Math.multiplyExact(value.epochSecond, 1_000_000L), value.nano.intdiv(NANOS_PER_MICRO)) } + catch (ArithmeticException failure) { throw new IllegalArgumentException("Column '$name', row $row: Instant overflows epoch micros", failure) } + } + Class current = policyType(value.getClass()) + if (current == null || (policy != null && policy != current)) { + throw new IllegalArgumentException("Column '$name', row $row: unsupported/mixed ${value.getClass().name}; expected ${policy?.name ?: 'supported encoding'}") + } + policy = current + } + if (policy == null) { + if (infer && !(declared in [List, Map])) { + if (!declared.isAssignableFrom(String)) { + throw new IllegalArgumentException("All-null column '$name' declared ${declared.name} requires explicit retyping") + } + policy = String + } else { + policy = policyType(declared) + } + } + if (policy == null) { + throw new IllegalArgumentException("Unsupported type ${declared.name} for column '$name'; convert explicitly") + } + Class metadata = infer || policy in [String, List, Map] ? policy : declared + if (!METADATA_TYPES.containsKey(metadata.name)) { + throw new IllegalArgumentException("Unsupported declared type ${declared.name} for column '$name'") + } + new ColumnTypePolicy(policy, metadata) + } + + static Class policyType(Class type) { + if (type == Character || Enum.isAssignableFrom(type) || CharSequence.isAssignableFrom(type)) { + return String + } + if (Collection.isAssignableFrom(type)) { + return List + } + if (Map.isAssignableFrom(type)) { + return Map + } + SUPPORTED.contains(type) ? type : null + } + + static Class box(Class type) { + if (!type.primitive) { + return type + } + [ (int): Integer, (long): Long, (float): Float, (double): Double, + (boolean): Boolean, (byte): Byte, (short): Short, (char): Character ][type] + } + + static int[] decimal(List values) { + int scale = 0 + int left = 1 + values.each { value -> + if (value instanceof BigDecimal) { + scale = Math.max(scale, value.scale()) + left = Math.max(left, value.precision() - value.scale()) + } + } + [left + scale, scale] as int[] + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DatasetBatches.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DatasetBatches.groovy new file mode 100644 index 000000000..0ce753e36 --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DatasetBatches.groovy @@ -0,0 +1,200 @@ +package se.alipsa.matrix.parquet + +import org.apache.parquet.conf.PlainParquetConfiguration +import org.apache.parquet.hadoop.ParquetFileReader +import org.apache.parquet.hadoop.metadata.ParquetMetadata +import org.apache.parquet.io.LocalInputFile + +import se.alipsa.matrix.core.Matrix + +import java.util.concurrent.Callable +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.Future + +/** Ordered shard batches with strict schema checks and at most one prefetched batch per worker. */ +final class DatasetBatches implements MatrixBatchIterator { + private final List files + private final List footers = [] + private final List> work = [] + private final List lengths = [] + private final List modified = [] + private final ParquetReadOptions options + private final ExecutorService executor + private final Map cursors = [:] + private final Map> pending = [:] + private int current + private int submitted + private long remaining + private volatile boolean closed + private Matrix template + private Matrix ready + + /** Checks every footer before reading any rows; supplied shard order is retained. */ + DatasetBatches(List selection, ParquetReadOptions options) { + options.validate() + this.options = options + if (selection == null) { throw new IllegalArgumentException('Dataset files cannot be null') } + files = selection*.canonicalFile + if (files.empty || files.toSet().size() != files.size()) { + throw new IllegalArgumentException('Dataset files must be nonempty and unique') + } + readFooters() + validateSchemas() + selectWork() + remaining = options.limit == null ? Long.MAX_VALUE : options.limit + ParquetReadOptions emptyOptions = ParquetReadOptions.fromMap(options.toMap()) + emptyOptions.rowGroups = [] + emptyOptions.sourceFileColumn = null + new ParquetBatches(new LocalInputFile(files[0].toPath()), emptyOptions, files[0].name, footers[0]).withCloseable { cursor -> template = cursor.emptyMatrix() } + if (!footers.every { it.fileMetaData.keyValueMetaData[MatrixParquetReader.METADATA_INDEX_COLUMNS] == footers[0].fileMetaData.keyValueMetaData[MatrixParquetReader.METADATA_INDEX_COLUMNS] }) { + template = Matrix.builder(template.matrixName).columnNames(template.columnNames()).types(template.types()).build() + } + if (options.sourceFileColumn != null) { template.addColumn(options.sourceFileColumn, String, []) } + executor = Executors.newFixedThreadPool(Math.min(options.parallelism, Math.max(1, work.size())), + { Runnable action -> + Thread worker = new Thread(action, 'matrix-parquet-dataset') + worker.daemon = true + worker + } as java.util.concurrent.ThreadFactory) + try { + if (remaining > 0) { (0.. + if (!file.isFile()) { throw new IllegalArgumentException("Not a dataset file: $file") } + lengths.add(file.length()) + modified.add(file.lastModified()) + def input = new LocalInputFile(file.toPath()) + def readOptions = org.apache.parquet.ParquetReadOptions.builder(new PlainParquetConfiguration()).build() + input.newStream().withCloseable { stream -> footers.add(ParquetFileReader.readFooter(input, readOptions, stream)) } + } + } + + private void validateSchemas() { + def schema = footers[0].fileMetaData.schema + footers.eachWithIndex { ParquetMetadata footer, int index -> + if (footer.fileMetaData.keyValueMetaData[MatrixParquetReader.METADATA_COLUMN_TYPES] != footers[0].fileMetaData.keyValueMetaData[MatrixParquetReader.METADATA_COLUMN_TYPES]) { throw new IllegalArgumentException("Incompatible shard Matrix types: ${files[index]}") } + if (footer.fileMetaData.schema.fields != schema.fields) { throw new IllegalArgumentException("Incompatible shard schema: ${files[index]}") } + if (options.sourceFileColumn != null && schema.containsField(options.sourceFileColumn)) { throw new IllegalArgumentException('sourceFileColumn collides with dataset schema') } + } + if (options.rowGroups != null) { throw new IllegalArgumentException('Dataset rowGroups require per-file reads') } + } + + private void selectWork() { + Set paths = files*.canonicalPath.toSet() + if (!paths.containsAll(options.rowGroupsByFile.keySet())) { throw new IllegalArgumentException('Unknown shard path in rowGroupsByFile') } + files.eachWithIndex { File file, int index -> + List ids = options.rowGroupsByFile.containsKey(file.canonicalPath) ? options.rowGroupsByFile[file.canonicalPath] : (0..= footers[index].blocks.size() }) { + throw new IllegalArgumentException("Invalid/duplicate row-group IDs for $file") + } + ids.each { Integer group -> work.add([file: index, group: group]) } + } + } + + @Override + Matrix emptyMatrix() { + Matrix result = Matrix.builder(template.matrixName).columnNames(template.columnNames()).types(template.types()).build() + if (template.hasIndex()) { result.createIndex(template.indexedColumns() as String[]) } + result + } + + private void scheduleWork() { + if (submitted >= work.size()) { return } + int index = submitted++ + int source = work[index].file + if (files[source].length() != lengths[source] || files[source].lastModified() != modified[source]) { + throw new IOException("Shard changed after footer validation: ${files[source]}") + } + ParquetReadOptions perFile = ParquetReadOptions.fromMap(options.toMap()) + perFile.limit = null + perFile.sourceFileColumn = null + perFile.rowGroups = [work[index].group] + cursors[index] = new ParquetBatches(new LocalInputFile(files[source].toPath()), perFile, files[source].name, footers[source]) + scheduleBatch(index) + } + + private void scheduleBatch(int index) { + ParquetBatches cursor = cursors[index] + pending[index] = executor.submit({ -> + synchronized (cursor) { + closed ? null : (cursor.hasNext() ? cursor.next() : null) + } + } as Callable) + } + + @Override + boolean hasNext() { + if (ready != null) { return true } + if (closed) { return false } + try { + while (remaining > 0 && current < work.size()) { + Matrix batch = pending.remove(current).get() + if (batch == null) { + cursors.remove(current).close() + current++ + scheduleWork() + continue + } + if (batch.rowCount() > remaining) { + List truncated = [] + batch.columnNames().each { truncated.add(new ArrayList(batch.column(it).subList(0, (int) remaining))) } + batch = Matrix.builder(batch.matrixName).columnNames(batch.columnNames()).types(batch.types()).columns(truncated).build() + } + remaining -= batch.rowCount() + if (options.sourceFileColumn != null) { batch.addColumn(options.sourceFileColumn, String, [files[work[current].file].absolutePath] * batch.rowCount()) } + batch = withDatasetIndex(batch) + ready = batch + if (remaining > 0) { scheduleBatch(current) } + return true + } + close() + false + } catch (Exception failure) { + String source = current < work.size() ? files[work[current].file].absolutePath : 'dataset' + try { close() } catch (Exception cleanup) { failure.addSuppressed(cleanup) } + throw new IOException("Dataset decoding failed for $source", failure) + } + } + + private Matrix withDatasetIndex(Matrix batch) { + if (template.hasIndex()) { + if (!batch.hasIndex()) { batch.createIndex(template.indexedColumns() as String[]) } + } else if (batch.hasIndex()) { + List columns = batch.columnNames().collect { (List) batch.column(it) } + return Matrix.builder(batch.matrixName).columnNames(batch.columnNames()).types(batch.types()) + .columns(columns).build() + } + batch + } + + @Override + Matrix next() { + if (!hasNext()) { throw new NoSuchElementException() } + Matrix result = ready + ready = null + result + } + + @Override + void close() { + if (closed) { return } + closed = true + pending.values().each { it.cancel(true) } + executor.shutdownNow() + try { + executor.awaitTermination(10, java.util.concurrent.TimeUnit.SECONDS) + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt() + } finally { + // A cancelled Future may still be leaving its decoder. Serialize close with its work. + cursors.values().each { ParquetBatches cursor -> + synchronized (cursor) { cursor.close() } + } + ready = null + } + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DecimalSchemaPolicy.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DecimalSchemaPolicy.groovy new file mode 100644 index 000000000..1708ddd83 --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DecimalSchemaPolicy.groovy @@ -0,0 +1,37 @@ +package se.alipsa.matrix.parquet + +import groovy.transform.PackageScope + +/** Shared exact-decimal settings for top-level and nested logical field paths. */ +@PackageScope +final class DecimalSchemaPolicy { + final Map settings + final boolean infer + final Integer precision + final Integer scale + final String path + final Set used + + DecimalSchemaPolicy(Map settings, boolean infer = true, Integer precision = null, + Integer scale = null, String path = '', Set used = new HashSet<>()) { + this.settings = settings + this.infer = infer + this.precision = precision + this.scale = scale + this.path = path + this.used = used + } + + DecimalSchemaPolicy child(String name) { + new DecimalSchemaPolicy(settings, infer, precision, scale, path.empty ? name : "$path.$name".toString(), used) + } + + int[] resolve(List values, int[] supplied) { + int[] explicit = settings[path] ?: supplied + if (settings.containsKey(path)) { used.add(path) } + if (explicit != null) { return explicit } + if (precision != null) { return [precision, scale] as int[] } + if (!infer) { throw new IllegalArgumentException("Decimal field '$path' requires explicit precision/scale when inference is disabled") } + ColumnTypePolicy.decimal(values) + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpRangeInputFile.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpRangeInputFile.groovy new file mode 100644 index 000000000..a425f2466 --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpRangeInputFile.groovy @@ -0,0 +1,166 @@ +package se.alipsa.matrix.parquet + +import org.apache.parquet.io.InputFile +import org.apache.parquet.io.SeekableInputStream + +import java.nio.ByteBuffer + +/** HTTP range input with a bounded per-stream cache and stable object validators. */ +final class HttpRangeInputFile implements InputFile { + private static final String CONTENT_RANGE = 'Content-Range' + private static final String ETAG = 'ETag' + private static final String LAST_MODIFIED = 'Last-Modified' + + private static final int MAX_REQUEST_BYTES = 8388608 + private static final int PREFETCH_BYTES = 65536 + private static final int EOF = -1 + private volatile URL resolvedTarget + private final URL source + private final ParquetReadOptions options + private final int cacheBytes + private final long length + private final String etag + private final String modified + + /** Verifies byte-range support immediately; errors allow AUTO callers to choose download. */ + HttpRangeInputFile(URL source, ParquetReadOptions options) { + if (!(source.protocol in ['http', 'https'])) { throw new IOException('Range reads require HTTP(S)') } + this.source = source + this.options = options + cacheBytes = options.rangeCacheBytes + + def response = request(0L, 0L) + def connection = response.connection + boolean complete = false + try { + String range = connection.getHeaderField(CONTENT_RANGE) + def match = range =~ /bytes 0-0\/(\d+)/ + if (!match.matches()) { throw new IOException("Invalid Content-Range: $range") } + length = Long.parseLong(match.group(1)) + if (length <= 0) { throw new IOException('Empty remote object') } + etag = connection.getHeaderField(ETAG) + modified = connection.getHeaderField(LAST_MODIFIED) + connection.inputStream.withCloseable { stream -> + if (stream.read() < 0 || stream.read() != EOF) { throw new IOException('Invalid range payload length') } + } + resolvedTarget = connection.URL + complete = true + } finally { if (!complete) { connection.disconnect() } } + } + + private Map request(long start, long end) { + Map headers = [Range: "bytes=$start-$end".toString()] + if (etag != null) { headers['If-Match'] = etag } + else if (modified != null) { headers['If-Unmodified-Since'] = modified } + URL target = resolvedTarget ?: source + HttpURLConnection connection = HttpTransport.open(target, options, headers) + if (target.toExternalForm() != source.toExternalForm() && connection.responseCode in [401, 403]) { + connection.disconnect() + connection = HttpTransport.open(source, options, headers) + } + if (connection.responseCode != 206 || !(connection.getHeaderField('Content-Encoding') in [null, 'identity'])) { + int status = connection.responseCode + connection.disconnect() + throw new IOException("Range request rejected: HTTP $status") + } + [connection: connection] + } + + private byte[] fetch(long start, int count) { + long end = start + count - 1 + HttpURLConnection connection = request(start, end).connection + boolean complete = false + try { + if (connection.getHeaderField(CONTENT_RANGE) != "bytes $start-$end/$length".toString() || + (etag != null && connection.getHeaderField(ETAG) != etag) || + (modified != null && connection.getHeaderField(LAST_MODIFIED) != modified)) { + throw new IOException('Remote object/range changed during read') + } + connection.inputStream.withCloseable { stream -> + byte[] bytes = stream.readNBytes(count) + if (bytes.length != count || stream.read() != EOF) { throw new EOFException('Truncated/oversized HTTP range') } + resolvedTarget = connection.URL + complete = true + bytes + } + } finally { if (!complete) { connection.disconnect() } } + } + + @Override + long getLength() { length } + + @Override + SeekableInputStream newStream() { + new SeekableInputStream() { + private long position + private long cacheStart = -1L + private byte[] cache = new byte[0] + private boolean closed + + @Override + long getPos() { position } + @Override + void seek(long value) { + if (value < 0 || value > length) { throw new EOFException("Invalid seek $value") } + position = value + } + @Override + int read() { + byte[] single = new byte[1] + read(single, 0, 1) < 0 ? EOF : single[0] & 0xff + } + @Override + int read(byte[] target, int offset, int count) { + Objects.checkFromIndexSize(offset, count, target.length) + if (closed) { throw new IOException('Stream closed') } + if (count == 0) { return 0 } + if (position >= length) { return EOF } + byte[] available = cache + long start = cacheStart + if (position < cacheStart || position >= cacheStart + cache.length) { + start = position + int requested = Math.max(Math.min(count, MAX_REQUEST_BYTES), Math.min(cacheBytes, PREFETCH_BYTES)) + available = fetch(position, (int) Math.min(requested, length - position)) + cacheStart = start + cache = available.length <= cacheBytes ? available : new byte[0] + } + int copied = (int) Math.min(count, start + available.length - position) + System.arraycopy(available, (int) (position - start), target, offset, copied) + position += copied + copied + } + @Override + void readFully(byte[] target) { readFully(target, 0, target.length) } + @Override + void readFully(byte[] target, int offset, int count) { + int total = 0 + while (total < count) { + int result = read(target, offset + total, count - total) + if (result < 0) { throw new EOFException() } + total += result + } + } + @Override + int read(ByteBuffer target) { + if (target.isReadOnly()) { throw new java.nio.ReadOnlyBufferException() } + if (target.hasArray()) { + int result = read(target.array(), target.arrayOffset() + target.position(), target.remaining()) + if (result > 0) { target.position(target.position() + result) } + return result + } + byte[] bytes = new byte[Math.min(target.remaining(), MAX_REQUEST_BYTES)] + int result = read(bytes, 0, bytes.length) + if (result > 0) { target.put(bytes, 0, result) } + result + } + @Override + void readFully(ByteBuffer target) { + while (target.hasRemaining()) { + if (read(target) < 0) { throw new EOFException() } + } + } + @Override + void close() { closed = true; cache = new byte[0] } + } + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpTransport.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpTransport.groovy new file mode 100644 index 000000000..1b6afb8c8 --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpTransport.groovy @@ -0,0 +1,60 @@ +package se.alipsa.matrix.parquet + +import groovy.transform.PackageScope + +/** Shared bounded redirects, timeouts and origin-scoped Hub credentials. */ +@PackageScope +final class HttpTransport { + private static final List PROTOCOLS = ['http', 'https'] + private static final String HTTPS = 'https' + private static final List HTTPS_PORTS = [-1, 443] + + private HttpTransport() {} + + static HttpURLConnection open(URL source, ParquetReadOptions options, Map headers) { + URL target = source + int redirects = 0 + int transientFailures = 0 + while (true) { + if (!PROTOCOLS.contains(target.protocol)) { throw new IOException('HTTP(S) URL required') } + HttpURLConnection connection = (HttpURLConnection) target.openConnection() + connection.instanceFollowRedirects = false + connection.connectTimeout = options.connectTimeoutMillis + connection.readTimeout = options.readTimeoutMillis + connection.setRequestProperty('Accept-Encoding', 'identity') + headers.each { String key, String value -> connection.setRequestProperty(key, value) } + if (options.hubToken != null && target.protocol == HTTPS && target.host == 'huggingface.co' && HTTPS_PORTS.contains(target.port)) { + connection.setRequestProperty('Authorization', "Bearer ${options.hubToken}") + } + int status + try { status = connection.responseCode } catch (IOException failure) { + connection.disconnect() + if (++transientFailures >= 2) { throw failure } + continue + } + transientFailures = 0 + if (!(status in [301, 302, 303, 307, 308])) { return connection } + String location = connection.getHeaderField('Location') + connection.disconnect() + if (location == null || redirects++ >= 5) { throw new IOException('Invalid/excessive redirects') } + URL next = new URL(target, location) + if (target.protocol == HTTPS && next.protocol != HTTPS) { throw new IOException('Unsafe HTTP redirect') } + target = next + } + } + + static InputStream download(URL source, ParquetReadOptions options) { + HttpURLConnection connection = open(source, options, [:]) + int status = connection.responseCode + if (status != 200) { + connection.disconnect() + throw new IOException("Download rejected: HTTP $status") + } + InputStream stream + try { stream = connection.inputStream } catch (IOException failure) { connection.disconnect(); throw failure } + new FilterInputStream(stream) { + @Override + void close() { try { super.close() } finally { connection.disconnect() } } + } + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HubUrl.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HubUrl.groovy new file mode 100644 index 000000000..1a8934fff --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HubUrl.groovy @@ -0,0 +1,66 @@ +package se.alipsa.matrix.parquet + +import java.nio.charset.StandardCharsets +import java.util.regex.Pattern + +/** Resolves namespaced dataset hf:// URIs, following huggingface_hub v2.2.0 parsing. */ +final class HubUrl { + private static final String SLASH = '/' + private static final String ERR_FILE_PATH = 'A dataset file path is required' + private static final String PLUS = '+' + private static final String PARENT = '..' + + private static final int PARTS = 3 + private static final String PREFIX = 'hf://datasets/' + private static final Pattern SPECIAL = Pattern.compile('^refs/(?:convert/[\\w.-]+|pr/\\d+)') + + private HubUrl() {} + + /** Resolves an hf://datasets/owner/repo[@revision]/path URI without contacting the Hub. */ + static URL resolve(String uri) { + if (uri == null || !uri.startsWith(PREFIX)) { throw new IllegalArgumentException('Expected hf://datasets/owner/repo[@revision]/path') } + String location = uri.substring(PREFIX.length()).replaceAll('^/+|/+$', '') + int at = location.indexOf('@') + String repo + String revision = 'main' + String path + if (at < 0 || location.substring(0, at).count(SLASH) > 1) { + String[] parts = location.split(SLASH, PARTS) + if (parts.length < PARTS) { throw new IllegalArgumentException(ERR_FILE_PATH) } + repo = parts[0] + SLASH + parts[1] + path = parts[2] + } else { + repo = location.substring(0, at) + String tail = location.substring(at + 1) + def special = SPECIAL.matcher(tail) + if (special.find()) { + revision = special.group() + path = tail.substring(revision.length()).replaceFirst('^/', '') + } else { + int slash = tail.indexOf(SLASH) + if (slash < 0) { throw new IllegalArgumentException(ERR_FILE_PATH) } + revision = tail.substring(0, slash) + path = tail.substring(slash + 1) + } + revision = URLDecoder.decode(revision.replace(PLUS, '%2B'), StandardCharsets.UTF_8) + } + if (!(repo ==~ /[^\/@]+\/[^\/@]+/) || revision.empty || path.empty) { + throw new IllegalArgumentException('Invalid dataset repository, revision or file path') + } + if (repo.contains('--') || repo.contains(PARENT) || repo.endsWith('.git') || + repo.split(SLASH).any { !(it ==~ /[\w](?:[\w.-]*[\w])?/) } || path.startsWith(SLASH) || path.contains('//') || hasDotSegments(path) || hasDotSegments(revision)) { + throw new IllegalArgumentException('Invalid repository id or empty path segment') + } + String encodedRevision = SPECIAL.matcher(revision).matches() ? revision : encode(revision) + String encodedPath = path.split(SLASH, -1).collect { encode(it) }.join(SLASH) + new URI("https://huggingface.co/datasets/$repo/resolve/$encodedRevision/$encodedPath").toURL() + } + + private static boolean hasDotSegments(String value) { + value.split(SLASH).any { it in ['.', PARENT] } + } + + private static String encode(String value) { + URLEncoder.encode(value, StandardCharsets.UTF_8).replace(PLUS, '%20').replace('%7E', '~').replace('*', '%2A') + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/InMemoryInputFile.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/InMemoryInputFile.groovy new file mode 100644 index 000000000..5b7551a09 --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/InMemoryInputFile.groovy @@ -0,0 +1,34 @@ +package se.alipsa.matrix.parquet + +import org.apache.parquet.io.DelegatingSeekableInputStream +import org.apache.parquet.io.InputFile +import org.apache.parquet.io.SeekableInputStream + +/** Seekable Parquet input backed by a byte array, without temporary files. */ +final class InMemoryInputFile implements InputFile { + private final byte[] content + + /** Retains the supplied array; callers must not modify it during a read. */ + InMemoryInputFile(byte[] content) { + if (content == null) { throw new IllegalArgumentException('content cannot be null') } + this.content = content + } + + @Override + long getLength() { content.length } + + @Override + SeekableInputStream newStream() { + def input = new ByteArrayInputStream(content) + new DelegatingSeekableInputStream(input) { + @Override + long getPos() { content.length - input.available() } + @Override + void seek(long position) { + if (position < 0 || position > content.length) { throw new EOFException("Invalid position $position") } + input.reset() + input.skipNBytes(position) + } + } + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixBatchIterator.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixBatchIterator.groovy new file mode 100644 index 000000000..cfc9668f8 --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixBatchIterator.groovy @@ -0,0 +1,9 @@ +package se.alipsa.matrix.parquet + +import se.alipsa.matrix.core.Matrix + +/** Closeable iterator of bounded Matrix batches; close explicitly on early termination. */ +interface MatrixBatchIterator extends Iterator, Closeable { + /** Returns an empty Matrix retaining this selection's schema and types. */ + Matrix emptyMatrix() +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetReader.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetReader.groovy index a2b19e7e1..e8b0e8f9e 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetReader.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetReader.groovy @@ -1,11 +1,8 @@ package se.alipsa.matrix.parquet -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path +import groovy.transform.PackageScope + import org.apache.parquet.example.data.Group -import org.apache.parquet.hadoop.ParquetFileReader -import org.apache.parquet.hadoop.ParquetReader -import org.apache.parquet.hadoop.example.GroupReadSupport import org.apache.parquet.schema.GroupType import org.apache.parquet.schema.LogicalTypeAnnotation import org.apache.parquet.schema.PrimitiveType @@ -15,16 +12,18 @@ import org.apache.parquet.schema.Type import se.alipsa.matrix.core.Matrix import se.alipsa.matrix.core.util.Logger +import java.nio.ByteBuffer import java.nio.file.Files import java.nio.file.StandardCopyOption import java.sql.Time +import java.sql.Timestamp import java.time.DateTimeException import java.time.Instant import java.time.LocalDate import java.time.LocalDateTime import java.time.LocalTime import java.time.ZoneId -import java.util.concurrent.ConcurrentHashMap +import java.time.ZoneOffset /** * Reads Apache Parquet files into {@link Matrix} objects. @@ -79,13 +78,16 @@ import java.util.concurrent.ConcurrentHashMap * *

Limitations

*
    - *
  • The entire file is loaded into memory (no streaming support)
    • + *
  • read() collects the result in memory; closeable batches bound accepted rows
    • *
* * @see MatrixParquetWriter * @see Matrix */ class MatrixParquetReader { + private static final String OPEN_BRACKET = '[' + private static final String EXTENSION = '.parquet' + private static final Logger log = Logger.getLogger(MatrixParquetReader) @@ -104,8 +106,8 @@ class MatrixParquetReader { /** Thread-local storage for timezone used during read operations */ private static final ThreadLocal ZONE_ID_HOLDER = new ThreadLocal<>() - /** Cache for Class.forName() results to avoid repeated reflection calls */ - private static final Map> CLASS_CACHE = new ConcurrentHashMap<>() + /** Binary text override used by the retained Group benchmark helper */ + private static final ThreadLocal BINARY_AS_STRING = new ThreadLocal<>() private static final String ERR_PATH_NULL = 'Path cannot be null' private static final String ERR_FILE_PATH_NULL = 'File path cannot be null' @@ -152,6 +154,12 @@ class MatrixParquetReader { new ReaderBuilder() } + /** Creates a reader from a validated snapshot of typed options. */ + static ReaderBuilder builder(ParquetReadOptions options) { + if (options == null) { throw new IllegalArgumentException('options cannot be null') } + new ReaderBuilder(options) + } + /** * Fluent builder for configuring and executing Parquet read operations. * @@ -159,9 +167,75 @@ class MatrixParquetReader { */ static class ReaderBuilder { - private final ParquetReadOptions options = new ParquetReadOptions() + private final ParquetReadOptions options + + private ReaderBuilder() { this(new ParquetReadOptions()) } + + private ReaderBuilder(ParquetReadOptions value) { + options = ParquetReadOptions.fromMap(value.toMap()) + options.hubToken = value.hubToken + } + + /** Sets columns; see ParquetReadOptions for defaults and validation. */ + ReaderBuilder columns(List value) { options.columns = value; this } + + /** Sets filter; see ParquetReadOptions for defaults and validation. */ + ReaderBuilder filter(ParquetPredicate value) { options.filter = value; this } + + /** Sets binaryAsString; see ParquetReadOptions for defaults and validation. */ + ReaderBuilder binaryAsString(Boolean value) { options.binaryAsString = value; this } + + /** Sets useColumnIndexFilter; see ParquetReadOptions for defaults and validation. */ + ReaderBuilder useColumnIndexFilter(boolean value) { options.useColumnIndexFilter = value; this } + + /** Sets limit; see ParquetReadOptions for defaults and validation. */ + ReaderBuilder limit(long value) { options.limit = value; this } + + /** Sets rowGroups; see ParquetReadOptions for defaults and validation. */ + ReaderBuilder rowGroups(List value) { options.rowGroups = value; this } + + /** Sets maxBatchRows; see ParquetReadOptions for defaults and validation. */ + ReaderBuilder maxBatchRows(int value) { options.maxBatchRows = value; this } + + /** Transport: DOWNLOAD, AUTO or RANGE; default 'DOWNLOAD'. */ + ReaderBuilder remoteReadMode(String value) { options.remoteReadMode = value; this } + + /** HTTP connection timeout; default 10000. */ + ReaderBuilder connectTimeoutMillis(int value) { options.connectTimeoutMillis = value; this } + + /** HTTP read timeout; default 60000. */ + ReaderBuilder readTimeoutMillis(int value) { options.readTimeoutMillis = value; this } + + /** Maximum HTTP range cache bytes; default 8388608. */ + ReaderBuilder rangeCacheBytes(int value) { options.rangeCacheBytes = value; this } + + /** Optional local source path column; default null. */ + ReaderBuilder sourceFileColumn(String value) { options.sourceFileColumn = value; this } + + /** Bounded dataset decoder workers; default 1. */ + ReaderBuilder parallelism(int value) { options.parallelism = value; this } + + /** Reads shards in caller order after checking every schema. */ + Matrix read(List files) { + batches(files).withCloseable { iterator -> collectBatches(iterator) } + } + + /** Opens ordered shard batches; prefetch is bounded by parallelism. */ + MatrixBatchIterator batches(List files) { new DatasetBatches(files, options) } + + /** Reads explicit file globs in normalized path order. */ + Matrix readGlob(String pattern) { read(globFiles(pattern)) } + + /** Selects original footer groups per shard, keyed by canonical local file path. */ + ReaderBuilder rowGroupsByFile(Map> value) { options.rowGroupsByFile = value; this } + + /** Opens closeable batches; close explicitly when stopping before exhaustion. */ + MatrixBatchIterator batches(File file) { + if (file?.isDirectory()) { return batches(directoryFiles(file)) } + validateFile(file) + new ParquetBatches(new org.apache.parquet.io.LocalInputFile(file.toPath()), options, defaultMatrixName(file), null, file.canonicalPath) + } - private ReaderBuilder() {} /** * Sets the name for the resulting Matrix. @@ -203,16 +277,9 @@ class MatrixParquetReader { * @return a Matrix populated with the file contents */ Matrix read(File file) { - if (options.zoneId != null && options.matrixName != null) { - return MatrixParquetReader.read(file, options.matrixName, options.zoneId) - } - if (options.zoneId != null) { - return MatrixParquetReader.read(file, options.zoneId) - } - if (options.matrixName != null) { - return MatrixParquetReader.read(file, options.matrixName) - } - MatrixParquetReader.read(file) + if (file?.isDirectory()) { return read(directoryFiles(file)) } + validateFile(file) + readInput(new org.apache.parquet.io.LocalInputFile(file.toPath()), options, defaultMatrixName(file), file.canonicalPath) } /** @@ -235,10 +302,7 @@ class MatrixParquetReader { * @return a Matrix populated with the byte array contents */ Matrix read(byte[] content) { - if (content == null) { - throw new IllegalArgumentException(ERR_CONTENT_NULL) - } - read(new ByteArrayInputStream(content)) + readInput(new InMemoryInputFile(content), options, null) } /** @@ -248,16 +312,12 @@ class MatrixParquetReader { * @return a Matrix populated with the stream contents */ Matrix read(InputStream is) { - if (options.zoneId != null && options.matrixName != null) { - return MatrixParquetReader.read(is, options.matrixName, options.zoneId) - } - if (options.zoneId != null) { - return MatrixParquetReader.read(is, options.zoneId) - } - if (options.matrixName != null) { - return MatrixParquetReader.read(is, options.matrixName) - } - MatrixParquetReader.read(is) + if (is == null) { throw new IllegalArgumentException('InputStream cannot be null') } + File temp = File.createTempFile('matrix-parquet-', EXTENSION) + try { + Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING) + readInput(new org.apache.parquet.io.LocalInputFile(temp.toPath()), options, null) + } finally { Files.deleteIfExists(temp.toPath()) } } /** @@ -267,19 +327,19 @@ class MatrixParquetReader { * @return a Matrix populated with the URL contents */ Matrix read(URL url) { - if (options.zoneId != null && options.matrixName != null) { - return MatrixParquetReader.read(url, options.matrixName, options.zoneId) - } - if (options.zoneId != null) { - return MatrixParquetReader.read(url, options.zoneId) - } - if (options.matrixName != null) { - url.openStream().withCloseable { InputStream is -> - MatrixParquetReader.read(is, options.matrixName) + if (url == null) { throw new IllegalArgumentException(ERR_URL_NULL) } + options.validate() + if (options.remoteReadMode != 'DOWNLOAD') { + HttpRangeInputFile input + try { input = new HttpRangeInputFile(url, options) } catch (IOException failure) { + if (options.remoteReadMode == 'RANGE') { throw failure } } - } else { - MatrixParquetReader.read(url) + if (input != null) { return readInput(input, options, null) } + } + if (url.protocol in ['http', 'https']) { + return HttpTransport.download(url, options).withCloseable { InputStream stream -> read(stream) } } + url.openStream().withCloseable { InputStream stream -> read(stream) } } /** @@ -305,6 +365,10 @@ class MatrixParquetReader { if (urlString == null) { throw new IllegalArgumentException(ERR_URL_STRING_NULL) } + if (urlString.startsWith('hf://')) { + options.hubToken = System.getenv('HF_TOKEN') + return read(HubUrl.resolve(urlString)) + } read(new URI(urlString).toURL()) } } @@ -318,21 +382,11 @@ class MatrixParquetReader { zoneId != null ? zoneId : ZoneId.systemDefault() } - /** - * Gets a cached Class object for the given class name. - * Uses Thread.currentThread().contextClassLoader.loadClass() and caches the result for subsequent calls. - */ - private static Class getCachedClass(String className) { - CLASS_CACHE.computeIfAbsent(className) { name -> - Thread.currentThread().contextClassLoader.loadClass(name) - } - } - /** * Validates the file parameter before reading. * * @param file the file to validate - * @throws IllegalArgumentException if file is null, does not exist, or is a directory + * @throws IllegalArgumentException if file is null or does not exist */ private static void validateFile(File file) { if (file == null) { @@ -359,10 +413,10 @@ class MatrixParquetReader { * @param file the Parquet file to read * @param matrixName the name to apply to the resulting matrix * @return a matrix populated with the file contents - * @throws IllegalArgumentException if file is null, does not exist, or is a directory + * @throws IllegalArgumentException if file is null or does not exist */ static Matrix read(File file, String matrixName) { - read(file, null, false, null).withMatrixName(matrixName) + builder().matrixName(matrixName).read(file) } /** @@ -370,7 +424,7 @@ class MatrixParquetReader { * * @param path the Path to read * @return a matrix populated with the file contents - * @throws IllegalArgumentException if path is null, does not exist, or is a directory + * @throws IllegalArgumentException if path is null or does not exist */ static Matrix read(java.nio.file.Path path) { if (path == null) { @@ -385,7 +439,7 @@ class MatrixParquetReader { * @param path the Path to read * @param matrixName the name to apply to the resulting matrix * @return a matrix populated with the file contents - * @throws IllegalArgumentException if path is null, does not exist, or is a directory + * @throws IllegalArgumentException if path is null or does not exist */ static Matrix read(java.nio.file.Path path, String matrixName) { if (path == null) { @@ -399,7 +453,7 @@ class MatrixParquetReader { * * @param filePath the file path to read * @return a matrix populated with the file contents - * @throws IllegalArgumentException if filePath is null, does not exist, or is a directory + * @throws IllegalArgumentException if filePath is null or does not exist */ static Matrix readFile(String filePath) { if (filePath == null) { @@ -414,7 +468,7 @@ class MatrixParquetReader { * @param filePath the file path to read * @param matrixName the name to apply to the resulting matrix * @return a matrix populated with the file contents - * @throws IllegalArgumentException if filePath is null, does not exist, or is a directory + * @throws IllegalArgumentException if filePath is null or does not exist */ static Matrix readFile(String filePath, String matrixName) { if (filePath == null) { @@ -426,7 +480,7 @@ class MatrixParquetReader { /** * Read Parquet content from a byte array into a {@link Matrix}. * - *

Parquet requires a seekable input, so the content is copied to a temporary file.

+ *

The byte array is read directly through a seekable in-memory input.

* * @param content byte array containing Parquet data * @return a matrix populated with the byte array contents @@ -436,13 +490,13 @@ class MatrixParquetReader { if (content == null) { throw new IllegalArgumentException(ERR_CONTENT_NULL) } - read(new ByteArrayInputStream(content)) + readInput(new InMemoryInputFile(content), new ParquetReadOptions(), null) } /** * Read Parquet content from a byte array into a {@link Matrix} using the supplied name. * - *

Parquet requires a seekable input, so the content is copied to a temporary file.

+ *

The byte array is read directly through a seekable in-memory input.

* * @param content byte array containing Parquet data * @param matrixName the name to apply to the resulting matrix @@ -453,13 +507,13 @@ class MatrixParquetReader { if (content == null) { throw new IllegalArgumentException(ERR_CONTENT_NULL) } - read(new ByteArrayInputStream(content), matrixName) + readInput(new InMemoryInputFile(content), new ParquetReadOptions().matrixName(matrixName), null) } /** * Read Parquet content from a byte array into a {@link Matrix} with a specific timezone. * - *

Parquet requires a seekable input, so the content is copied to a temporary file.

+ *

The byte array is read directly through a seekable in-memory input.

* * @param content byte array containing Parquet data * @param zoneId the timezone to use for converting UTC timestamps to LocalDateTime values @@ -470,13 +524,13 @@ class MatrixParquetReader { if (content == null) { throw new IllegalArgumentException(ERR_CONTENT_NULL) } - read(new ByteArrayInputStream(content), zoneId) + readInput(new InMemoryInputFile(content), new ParquetReadOptions().zoneId(zoneId), null) } /** * Read Parquet content from a byte array into a {@link Matrix} with a specific timezone and name. * - *

Parquet requires a seekable input, so the content is copied to a temporary file.

+ *

The byte array is read directly through a seekable in-memory input.

* * @param content byte array containing Parquet data * @param matrixName the name to apply to the resulting matrix @@ -488,7 +542,7 @@ class MatrixParquetReader { if (content == null) { throw new IllegalArgumentException(ERR_CONTENT_NULL) } - read(new ByteArrayInputStream(content), matrixName, zoneId) + readInput(new InMemoryInputFile(content), new ParquetReadOptions().matrixName(matrixName).zoneId(zoneId), null) } /** @@ -529,9 +583,7 @@ class MatrixParquetReader { if (url == null) { throw new IllegalArgumentException(ERR_URL_NULL) } - url.openStream().withCloseable { InputStream is -> - readFromInputStream(is, null, null) - } + builder().read(url) } /** @@ -545,7 +597,7 @@ class MatrixParquetReader { if (urlString == null) { throw new IllegalArgumentException(ERR_URL_STRING_NULL) } - read(new URI(urlString).toURL()) + builder().readUrl(urlString) } /** @@ -557,13 +609,13 @@ class MatrixParquetReader { * @param file the Parquet file to read * @param zoneId the timezone to use for converting UTC timestamps to LocalDateTime values * @return a matrix populated with the file contents - * @throws IllegalArgumentException if file is null, does not exist, is a directory, or zoneId is null + * @throws IllegalArgumentException if file is null, does not exist, or zoneId is null */ static Matrix read(File file, ZoneId zoneId) { if (zoneId == null) { throw new IllegalArgumentException(ERR_ZONE_ID_NULL) } - read(file, file == null ? null : defaultMatrixName(file), false, zoneId) + builder().zoneId(zoneId).read(file) } /** @@ -572,7 +624,7 @@ class MatrixParquetReader { * @param path the Path to read * @param zoneId the timezone to use for converting UTC timestamps to LocalDateTime values * @return a matrix populated with the file contents - * @throws IllegalArgumentException if path is null, does not exist, or is a directory, or zoneId is null + * @throws IllegalArgumentException if path is null, does not exist, or zoneId is null */ static Matrix read(java.nio.file.Path path, ZoneId zoneId) { if (path == null) { @@ -628,9 +680,7 @@ class MatrixParquetReader { if (zoneId == null) { throw new IllegalArgumentException(ERR_ZONE_ID_NULL) } - url.openStream().withCloseable { InputStream is -> - readFromInputStream(is, null, zoneId) - } + builder().zoneId(zoneId).read(url) } /** @@ -645,7 +695,8 @@ class MatrixParquetReader { if (urlString == null) { throw new IllegalArgumentException(ERR_URL_STRING_NULL) } - read(new URI(urlString).toURL(), zoneId) + if (zoneId == null) { throw new IllegalArgumentException(ERR_ZONE_ID_NULL) } + builder().zoneId(zoneId).readUrl(urlString) } /** @@ -655,13 +706,13 @@ class MatrixParquetReader { * @param matrixName the name to apply to the resulting matrix * @param zoneId the timezone to use for converting UTC timestamps to LocalDateTime values * @return a matrix populated with the file contents - * @throws IllegalArgumentException if file is null, does not exist, is a directory, or zoneId is null + * @throws IllegalArgumentException if file is null, does not exist, or zoneId is null */ static Matrix read(File file, String matrixName, ZoneId zoneId) { if (zoneId == null) { throw new IllegalArgumentException(ERR_ZONE_ID_NULL) } - read(file, null, false, zoneId).withMatrixName(matrixName) + builder().matrixName(matrixName).zoneId(zoneId).read(file) } /** @@ -730,9 +781,7 @@ class MatrixParquetReader { if (zoneId == null) { throw new IllegalArgumentException(ERR_ZONE_ID_NULL) } - url.openStream().withCloseable { InputStream is -> - readFromInputStream(is, matrixName, zoneId) - } + builder().matrixName(matrixName).zoneId(zoneId).read(url) } /** @@ -748,7 +797,8 @@ class MatrixParquetReader { if (urlString == null) { throw new IllegalArgumentException(ERR_URL_STRING_NULL) } - read(new URI(urlString).toURL(), matrixName, zoneId) + if (zoneId == null) { throw new IllegalArgumentException(ERR_ZONE_ID_NULL) } + builder().matrixName(matrixName).zoneId(zoneId).readUrl(urlString) } /** @@ -756,78 +806,70 @@ class MatrixParquetReader { * * @param file the Parquet file to read * @return a matrix populated with the file contents - * @throws IllegalArgumentException if file is null, does not exist, or is a directory + * @throws IllegalArgumentException if file is null or does not exist */ static Matrix read(File file) { - read(file, file == null ? null : defaultMatrixName(file), false, null) + if (file?.isDirectory()) { return builder().read(file) } + builder().read(file) } - private static Matrix read(File file, String fallbackMatrixName, boolean preferSchemaName, ZoneId zoneId) { - if (zoneId == null) { - return read(file, fallbackMatrixName, preferSchemaName) - } - try { - ZONE_ID_HOLDER.set(zoneId) - return read(file, fallbackMatrixName, preferSchemaName) - } finally { - ZONE_ID_HOLDER.remove() + /** Reads a list of shards in the supplied order, with strict schema compatibility. */ + static Matrix read(List files) { builder().read(files) } + + /** Reads an explicit glob; ordinary file strings are not interpreted as patterns. */ + static Matrix readGlob(String pattern) { builder().readGlob(pattern) } + + private static List directoryFiles(File directory) { + (directory.listFiles() ?: new File[0]).findAll { it.isFile() && it.name.endsWith(EXTENSION) } + .sort { it.canonicalPath }.toList() + } + + private static List globFiles(String pattern) { + java.nio.file.Path path = java.nio.file.Path.of(pattern).toAbsolutePath().normalize() + String text = path as String + int wildcard = text.findIndexOf { it in ['*', '?', OPEN_BRACKET, '{'] } + if (wildcard < 0) { return [path.toFile()] } + int slash = text.lastIndexOf(File.separator, wildcard) + java.nio.file.Path root = java.nio.file.Path.of(text.substring(0, Math.max(1, slash))) + def matcher = java.nio.file.FileSystems.default.getPathMatcher("glob:$text") + Files.walk(root).withCloseable { paths -> + paths.filter { Files.isRegularFile(it) && matcher.matches(it) }.sorted().map { it.toFile() }.toList() } } - private static Matrix read(File file, String fallbackMatrixName, boolean preferSchemaName) { - validateFile(file) - log.debug("Reading Parquet file: ${file.absolutePath}") - def path = new Path(file.toURI()) - def conf = new Configuration() - // Open Parquet file metadata - def footer = ParquetFileReader.readFooter(conf, path) - def keyValueMetaData = footer.getFileMetaData().getKeyValueMetaData() - def typeString = keyValueMetaData.get(METADATA_COLUMN_TYPES) - def indexString = keyValueMetaData.get(METADATA_INDEX_COLUMNS) - - List fieldTypes - if (typeString != null) { - fieldTypes = parseTypeString(typeString) - } - - ParquetReader.builder(new GroupReadSupport(), path).build().withCloseable { reader -> - String schemaName = footer.getFileMetaData().getSchema().name - String matrixName = fallbackMatrixName - if (preferSchemaName && schemaName != null && !schemaName.trim().isEmpty()) { - matrixName = schemaName - } + private static Matrix collectBatches(MatrixBatchIterator iterator) { + Matrix prototype = iterator.emptyMatrix() + List columns = [] + prototype.columnNames().each { columns.add([]) } + while (iterator.hasNext()) { + Matrix batch = iterator.next() + batch.columnNames().eachWithIndex { String name, int index -> columns[index].addAll(batch.column(name)) } + } + Matrix result = Matrix.builder(prototype.matrixName).columnNames(prototype.columnNames()).types(prototype.types()).columns(columns).build() + if (prototype.hasIndex()) { result.createIndex(prototype.indexedColumns() as String[]) } + result + } - Group row = reader.read() - GroupType schema = row != null ? row.getType() : footer.getFileMetaData().getSchema() - List fieldNames = schema.fields*.name - if (fieldTypes == null) { - fieldTypes = extractFieldTypes(schema) - } - if (row == null) { - Matrix m = Matrix.builder(matrixName) - .columnNames(fieldNames) - .types(fieldTypes) - .build() - return restoreIndex(m, indexString) - } + @PackageScope + static T withReadZone(ZoneId zone, Closure action) { + ZoneId previous = ZONE_ID_HOLDER.get() + if (zone != null) { ZONE_ID_HOLDER.set(zone) } + try { action.call() } finally { + if (previous == null) { ZONE_ID_HOLDER.remove() } else { ZONE_ID_HOLDER.set(previous) } + } + } - def builder = Matrix.builder(matrixName) - .columnNames(fieldNames) - .types(fieldTypes) - - while (row != null) { - def rowData = [] - fieldNames.eachWithIndex { name, i -> - def type = fieldTypes[i] - def field = schema.getType(name) - def value = readValue(row, name, field, type) - rowData << value - } - builder.addRow(rowData as Object[]) - row = reader.read() + @PackageScope + static Matrix readInput(org.apache.parquet.io.InputFile input, ParquetReadOptions options, String name, String sourcePath = null) { + log.debug("Reading Parquet input: $name") + ZoneId previous = ZONE_ID_HOLDER.get() + if (options.zoneId != null) { ZONE_ID_HOLDER.set(options.zoneId) } + try { + new ParquetBatches(input, options, name, null, sourcePath).withCloseable { batches -> + collectBatches(batches) } - Matrix m = builder.build() - restoreIndex(m, indexString) + } finally { + if (previous == null) { ZONE_ID_HOLDER.remove() } else { ZONE_ID_HOLDER.set(previous) } } } @@ -838,16 +880,18 @@ class MatrixParquetReader { * @param indexString JSON array or legacy comma-separated index column names, or null if no index * @return the matrix with index restored (or unchanged if no index metadata) */ - private static Matrix restoreIndex(Matrix matrix, String indexString) { + @PackageScope + static Matrix restoreIndex(Matrix matrix, String indexString) { if (indexString != null && !indexString.trim().isEmpty()) { - matrix.createIndex(parseIndexColumns(indexString)) + String[] indexes = parseIndexColumns(indexString) + if (indexes.every { matrix.columnNames().contains(it) }) { matrix.createIndex(indexes) } } matrix } private static String[] parseIndexColumns(String indexString) { String value = indexString.trim() - if (value.startsWith('[')) { + if (value.startsWith(OPEN_BRACKET)) { try { return parseJsonStringArray(value) as String[] } catch (IllegalArgumentException ignored) { @@ -864,7 +908,7 @@ class MatrixParquetReader { private static List parseJsonStringArray(String json) { List result = [] int pos = skipWhitespace(json, 0) - if (pos >= json.length() || json.charAt(pos) != '[' as char) { + if (pos >= json.length() || json.charAt(pos) != OPEN_BRACKET as char) { throw new IllegalArgumentException("Invalid index metadata JSON: $json") } pos = skipWhitespace(json, pos + 1) @@ -952,30 +996,7 @@ class MatrixParquetReader { } private static Matrix readFromInputStream(InputStream is, String matrixName, ZoneId zoneId) { - if (is == null) { - throw new IllegalArgumentException('InputStream cannot be null') - } - java.nio.file.Path tempFile = Files.createTempFile('matrix-parquet-', '.parquet') - try { - Files.copy(is, tempFile, StandardCopyOption.REPLACE_EXISTING) - File file = tempFile.toFile() - Matrix result = read(file, defaultMatrixName(file), true, zoneId) - if (matrixName != null && !matrixName.trim().isEmpty()) { - result = result.withMatrixName(matrixName) - } - result - } finally { - try { - Files.deleteIfExists(tempFile) - } catch (IOException ignored) { - } - } - } - - private static List parseTypeString(String typeString) { - typeString.split(COMMA).collect { className -> - getCachedClass(className.trim()) - } + builder().matrixName(matrixName).zoneId(zoneId).read(is) } private static Class getJavaType(PrimitiveType.PrimitiveTypeName typeName) { @@ -985,11 +1006,20 @@ class MatrixParquetReader { case PrimitiveTypeName.FLOAT -> Float case PrimitiveTypeName.DOUBLE -> Double case PrimitiveTypeName.BOOLEAN -> Boolean + case PrimitiveTypeName.BINARY, PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY -> byte[] default -> String } } - private static List extractFieldTypes(GroupType schema) { + @PackageScope + static boolean isTextLogicalType(LogicalTypeAnnotation logical) { + logical instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation || + logical instanceof LogicalTypeAnnotation.JsonLogicalTypeAnnotation || + logical instanceof LogicalTypeAnnotation.EnumLogicalTypeAnnotation + } + + @PackageScope + static List extractFieldTypes(GroupType schema) { schema.fields.collect { field -> def logical = field.getLogicalTypeAnnotation() if (logical instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) { @@ -1010,6 +1040,10 @@ class MatrixParquetReader { BigDecimal } else if (logical instanceof LogicalTypeAnnotation.TimeLogicalTypeAnnotation) { Time + } else if (isTextLogicalType(logical)) { + String + } else if (logical instanceof LogicalTypeAnnotation.UUIDLogicalTypeAnnotation) { + UUID } else { getJavaType(primitive) } @@ -1020,7 +1054,8 @@ class MatrixParquetReader { } } - private static Object readValue(Group group, String fieldName, Type fieldType, Class expectedType) { + @PackageScope + static Object readValue(Group group, String fieldName, Type fieldType, Class expectedType) { if (group.getFieldRepetitionCount(fieldName) == 0) { return null } @@ -1041,63 +1076,78 @@ class MatrixParquetReader { } private static Object readPrimitive(Group group, String fieldName, PrimitiveType fieldType, Class expectedType) { - def logical = fieldType.logicalTypeAnnotation - switch (fieldType.primitiveTypeName) { - case PrimitiveTypeName.INT32 -> { - if (logical == LogicalTypeAnnotation.dateType()) { - LocalDate.ofEpochDay(group.getInteger(fieldName, 0)) - } else if (logical instanceof LogicalTypeAnnotation.TimeLogicalTypeAnnotation) { - readTime(fieldName, group.getInteger(fieldName, 0), logical as LogicalTypeAnnotation.TimeLogicalTypeAnnotation) - } else if (expectedType == java.sql.Date) { - java.sql.Date.valueOf(LocalDate.ofEpochDay(group.getInteger(fieldName, 0))) - } else { - group.getInteger(fieldName, 0) - } - } - case PrimitiveTypeName.INT64 -> { - if (logical instanceof LogicalTypeAnnotation.TimeLogicalTypeAnnotation) { - readTime(fieldName, group.getLong(fieldName, 0), logical as LogicalTypeAnnotation.TimeLogicalTypeAnnotation) - } else if (logical instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { - Instant instant = timestampInstant(group.getLong(fieldName, 0), logical.unit) - if (expectedType == java.sql.Timestamp) { - java.sql.Timestamp.from(instant) - } else if (expectedType == Date) { - Date.from(instant) - } else { - LocalDateTime.ofInstant(instant, getZoneId()) - } - } else { - def longValue = group.getLong(fieldName, 0) - if (expectedType == BigInteger) { - BigInteger.valueOf(longValue) - } else if (expectedType == Date) { - new Date(longValue) - } else { - longValue - } - } - } + Object raw = switch (fieldType.primitiveTypeName) { + case PrimitiveTypeName.INT32 -> group.getInteger(fieldName, 0) + case PrimitiveTypeName.INT64 -> group.getLong(fieldName, 0) case PrimitiveTypeName.FLOAT -> group.getFloat(fieldName, 0) case PrimitiveTypeName.DOUBLE -> group.getDouble(fieldName, 0) case PrimitiveTypeName.BOOLEAN -> group.getBoolean(fieldName, 0) - case PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, PrimitiveTypeName.BINARY -> { - if (logical instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { - def scale = logical.scale - def binary = group.getBinary(fieldName, 0) - def unscaled = new BigInteger(binary.getBytes()) - if (expectedType == BigInteger) { - unscaled - } else { - new BigDecimal(unscaled, scale) - } - } else if (expectedType == BigDecimal) { - new BigDecimal(group.getBinary(fieldName, 0).toStringUsingUTF8()) - } else { - group.getBinary(fieldName, 0).toStringUsingUTF8() - } + case PrimitiveTypeName.BINARY, PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY -> group.getBinary(fieldName, 0) + default -> throw new IllegalArgumentException("Unsupported ${fieldType.primitiveTypeName} field $fieldName") + } + decodePrimitive(raw, fieldType, expectedType, BINARY_AS_STRING.get() == Boolean.TRUE) + } + + @PackageScope + static Object decodePrimitive(Object raw, PrimitiveType field, Class expected, boolean binaryAsString) { + if (field.primitiveTypeName == PrimitiveTypeName.INT96) { + throw new IllegalArgumentException("Unsupported INT96 field '${field.name}'; convert legacy timestamps explicitly") + } + def logical = field.logicalTypeAnnotation + if (logical instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { + BigInteger unscaled = raw instanceof org.apache.parquet.io.api.Binary ? new BigInteger(raw.bytes) : BigInteger.valueOf(((Number) raw).longValue()) + return expected == BigInteger ? unscaled : new BigDecimal(unscaled, logical.scale) + } + if (logical instanceof LogicalTypeAnnotation.DateLogicalTypeAnnotation) { + LocalDate date = LocalDate.ofEpochDay(((Number) raw).longValue()) + return expected in [java.sql.Date, Date] ? java.sql.Date.valueOf(date) : date + } + if (logical instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { + Instant instant = timestampInstant(((Number) raw).longValue(), logical.unit) + if (expected == Instant) { return instant } + if (expected == Timestamp) { return Timestamp.from(instant) } + if (expected == Date) { return Date.from(instant) } + return LocalDateTime.ofInstant(instant, logical.isAdjustedToUTC() ? getZoneId() : ZoneOffset.UTC) + } + if (logical instanceof LogicalTypeAnnotation.TimeLogicalTypeAnnotation) { + long value = ((Number) raw).longValue() + if (expected == LocalTime || (expected == null && !logical.isAdjustedToUTC())) { + long multiplier = logical.unit == LogicalTypeAnnotation.TimeUnit.MILLIS ? NANOS_PER_MILLI : + logical.unit == LogicalTypeAnnotation.TimeUnit.MICROS ? NANOS_PER_MICRO : 1L + return LocalTime.ofNanoOfDay(Math.multiplyExact(value, multiplier)) } - default -> group.getString(fieldName, 0) + return readTime(field.name, value, logical) + } + if (logical instanceof LogicalTypeAnnotation.UUIDLogicalTypeAnnotation) { + ByteBuffer buffer = ByteBuffer.wrap(((org.apache.parquet.io.api.Binary) raw).bytes) + return new UUID(buffer.getLong(), buffer.getLong()) + } + if (logical instanceof LogicalTypeAnnotation.Float16LogicalTypeAnnotation) { + throw new IllegalArgumentException("Unsupported FLOAT16 field '${field.name}'") + } + if (raw instanceof org.apache.parquet.io.api.Binary) { + if (expected == BigDecimal) { return new BigDecimal(raw.toStringUsingUTF8()) } + if (isTextLogicalType(logical) || expected == String || + (logical == null && field.primitiveTypeName == PrimitiveTypeName.BINARY && binaryAsString && expected != byte[])) { + return raw.toStringUsingUTF8() + } + return raw.bytes + } + narrowInteger(raw, field.name, expected) + } + + private static Object narrowInteger(Object raw, String fieldName, Class expected) { + if (expected == Byte) { + long value = ((Number) raw).longValue() + if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) { throw new IllegalArgumentException("Byte field '$fieldName' is out of range") } + return ((Number) raw).byteValue() + } + if (expected == Short) { + long value = ((Number) raw).longValue() + if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { throw new IllegalArgumentException("Short field '$fieldName' is out of range") } + return ((Number) raw).shortValue() } + raw } private static Time readTime(String fieldName, long value, @@ -1129,9 +1179,7 @@ class MatrixParquetReader { time.setTime(time.time + millisecondRemainder) time } catch (DateTimeException exception) { - // Unreachable in practice: the one-day range guard above rejects out-of-range values - // before LocalTime.ofNanoOfDay can throw, so the contextual error already comes from - // the guard. Kept defensively in case a future unit change bypasses it. + // Guard defensively in case a future unit bypasses the one-day range check. throw new IllegalArgumentException("Invalid TIME value $value for field '$fieldName'", exception) } } diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy index 4e7801eb3..351b683ff 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy @@ -1,6 +1,5 @@ package se.alipsa.matrix.parquet -import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.parquet.example.data.Group import org.apache.parquet.example.data.simple.SimpleGroupFactory @@ -22,8 +21,10 @@ import java.beans.PropertyDescriptor import java.math.RoundingMode import java.sql.Time import java.sql.Timestamp +import java.time.Instant import java.time.LocalDate import java.time.LocalDateTime +import java.time.LocalTime import java.time.ZoneId /** @@ -116,6 +117,33 @@ class MatrixParquetWriter { static class WriterBuilder { private final Matrix matrix + /** Sets rowGroupSize; see ParquetWriteOptions for defaults. */ + WriterBuilder rowGroupSize(long value) { options.rowGroupSize(value); this } + + /** Sets pageSize; see ParquetWriteOptions for defaults. */ + WriterBuilder pageSize(int value) { options.pageSize(value); this } + + /** Sets dictionaryEncoding; see ParquetWriteOptions for defaults. */ + WriterBuilder dictionaryEncoding(boolean value) { options.dictionaryEncoding(value); this } + + /** Sets writerVersion; see ParquetWriteOptions for defaults. */ + WriterBuilder writerVersion(String value) { options.writerVersion(value); this } + + /** Sets columnIndexTruncateLength; see ParquetWriteOptions for defaults. */ + WriterBuilder columnIndexTruncateLength(int value) { options.columnIndexTruncateLength(value); this } + + /** Sets statisticsTruncateLength; see ParquetWriteOptions for defaults. */ + WriterBuilder statisticsTruncateLength(int value) { options.statisticsTruncateLength(value); this } + + /** Sets bloomFilterEnabled; see ParquetWriteOptions for defaults. */ + WriterBuilder bloomFilterEnabled(Map value) { options.bloomFilterEnabled(value); this } + + /** Sets bloomFilterNDV; see ParquetWriteOptions for defaults. */ + WriterBuilder bloomFilterNDV(Map value) { options.bloomFilterNDV(value); this } + + /** Sets sizeStatisticsEnabled; see ParquetWriteOptions for defaults. */ + WriterBuilder sizeStatisticsEnabled(boolean value) { options.sizeStatisticsEnabled(value); this } + private final ParquetWriteOptions options = new ParquetWriteOptions() private WriterBuilder(Matrix matrix) { @@ -304,14 +332,14 @@ class MatrixParquetWriter { * @param matrix the matrix to write * @param fileOrDir the target file or directory. If directory the file name will be based on the matrix name. * @param inferPrecisionAndScale whether to infer precision and scale for BigDecimal columns. If false, - * BigDecimal columns default to double storage (which leads to some loss of precision). + * BigDecimal fields require explicit precision/scale settings. * Nested Groovy {@link List} and {@link Map} structures are preserved as Parquet LIST/MAP/STRUCT types. * @return the target file * @throws IllegalArgumentException if matrix is null, has no columns, or fileOrDir is null */ static File write(Matrix matrix, File fileOrDir, boolean inferPrecisionAndScale = true) { validateInput(matrix, fileOrDir) - MessageType schema + WriteSchema schema if (inferPrecisionAndScale) { schema = buildSchema(matrix,true) } else { @@ -421,13 +449,13 @@ class MatrixParquetWriter { } options.validate() File file = determineTargetFile(matrix, fileOrDir) - MessageType schema = createSchema(matrix, options) + WriteSchema schema = createSchema(matrix, options) if (options.zoneId != null) { return ParquetValueWriter.withZoneId(options.zoneId) { - writeInternal(matrix, file, schema, options.compressionCodec, options.roundingMode) + writeInternal(matrix, file, schema, options.compressionCodec, options.roundingMode, options) } } - return writeInternal(matrix, file, schema, options.compressionCodec, options.roundingMode) + return writeInternal(matrix, file, schema, options.compressionCodec, options.roundingMode, options) } /** @@ -435,13 +463,13 @@ class MatrixParquetWriter { * * @param matrix the matrix to write * @param inferPrecisionAndScale whether to infer precision and scale for BigDecimal columns. If false, - * BigDecimal columns default to double storage (which leads to some loss of precision). + * BigDecimal fields require explicit precision/scale settings. * @return byte array containing Parquet data * @throws IllegalArgumentException if matrix is null or has no columns */ static byte[] writeBytes(Matrix matrix, boolean inferPrecisionAndScale = true) { validateInput(matrix) - MessageType schema = buildSchema(matrix, inferPrecisionAndScale) + WriteSchema schema = buildSchema(matrix, inferPrecisionAndScale) return writeBytesInternal(matrix, schema) } @@ -470,7 +498,7 @@ class MatrixParquetWriter { */ static byte[] writeBytes(Matrix matrix, Map decimalMeta) { validateInput(matrix) - MessageType schema = buildSchema(matrix, ParquetWriteOptions.normalizeDecimalMeta(decimalMeta ?: [:])) + WriteSchema schema = buildSchema(matrix, ParquetWriteOptions.normalizeDecimalMeta(decimalMeta ?: [:])) return writeBytesInternal(matrix, schema) } @@ -487,13 +515,13 @@ class MatrixParquetWriter { throw new IllegalArgumentException(ERR_OPTIONS_NULL) } options.validate() - MessageType schema = createSchema(matrix, options) + WriteSchema schema = createSchema(matrix, options) if (options.zoneId != null) { return ParquetValueWriter.withZoneId(options.zoneId) { - writeBytesInternal(matrix, schema, options.compressionCodec, options.roundingMode) + writeBytesInternal(matrix, schema, options.compressionCodec, options.roundingMode, options) } } - return writeBytesInternal(matrix, schema, options.compressionCodec, options.roundingMode) + return writeBytesInternal(matrix, schema, options.compressionCodec, options.roundingMode, options) } private static File determineTargetFile(Matrix matrix, File fileOrDir) { @@ -515,26 +543,15 @@ class MatrixParquetWriter { String normalized = (matrixName ?: DEFAULT_MATRIX_NAME) .replaceAll(/[\\\/:<>"|?*\p{Cntrl}]/, '_') .replaceAll(/[. ]+$/, '') - if (normalized.trim().isEmpty() || normalized.replace('.', '').trim().isEmpty()) { + if (normalized.trim().isEmpty() || normalized.replace(DOT, '').trim().isEmpty()) { normalized = DEFAULT_MATRIX_NAME } normalized.toLowerCase(Locale.ROOT).endsWith('.parquet') ? normalized : "${normalized}.parquet" } - private static MessageType createSchema(Matrix matrix, ParquetWriteOptions options) { - if (options.hasDecimalMeta()) { - return buildSchema(matrix, options.decimalMeta) - } - if (options.hasUniformPrecisionAndScale()) { - Map decimalMeta = [:] - matrix.columnNames().each { String col -> - if (matrix.type(col) == BigDecimal) { - decimalMeta[col] = [options.precision, options.scale] as int[] - } - } - return buildSchema(matrix, decimalMeta) - } - return buildSchema(matrix, options.inferPrecisionAndScale) + private static WriteSchema createSchema(Matrix matrix, ParquetWriteOptions options) { + buildSchema(matrix, options.decimalMeta, + new DecimalSchemaPolicy(options.decimalMeta, options.inferPrecisionAndScale, options.precision, options.scale)) } /** @@ -542,26 +559,67 @@ class MatrixParquetWriter { * to a Parquet file, given an already constructed schema. * @param matrix the matrix to write * @param file the target file - * @param schema the Parquet MessageType schema + * @param resolved the validated physical schema and column policies * @return the target file */ - private static File writeInternal(Matrix matrix, File file, MessageType schema, - CompressionCodecName compressionCodec = CompressionCodecName.SNAPPY, RoundingMode roundingMode = RoundingMode.UNNECESSARY) { - def conf = new Configuration() - def writer = ExampleParquetWriter.builder(new Path(file.toURI())) - .withConf(conf) - .withWriteMode(ParquetFileWriter.Mode.OVERWRITE) - .withType(schema) - .withCompressionCodec(compressionCodec) - .withExtraMetaData(buildExtraMeta(matrix)) - .build() - writer.withCloseable { parquetWriter -> writeRows(parquetWriter, matrix, schema, roundingMode) } - return file + private static File writeInternal(Matrix matrix, File file, WriteSchema resolved, + CompressionCodecName compressionCodec = CompressionCodecName.SNAPPY, RoundingMode roundingMode = RoundingMode.UNNECESSARY, ParquetWriteOptions options = new ParquetWriteOptions()) { + MessageType schema = resolved.schema + java.nio.file.Path target = resolveOutputTarget(file.toPath().toAbsolutePath()) + java.nio.file.Path temporary = target.resolveSibling(".matrix-parquet-${UUID.randomUUID()}.tmp") + java.nio.file.Files.createFile(temporary) + try { + if (java.nio.file.Files.exists(target) && java.nio.file.Files.getFileStore(target).supportsFileAttributeView('posix')) { + java.nio.file.Files.setPosixFilePermissions(temporary, java.nio.file.Files.getPosixFilePermissions(target)) + } + def conf = new org.apache.parquet.conf.PlainParquetConfiguration() + def writerBuilder = ExampleParquetWriter.builder(new org.apache.parquet.io.LocalOutputFile(temporary)) + .withConf(conf).withWriteMode(ParquetFileWriter.Mode.OVERWRITE).withType(schema) + .withCompressionCodec(compressionCodec).withExtraMetaData(buildExtraMeta(matrix, resolved)) + def writer = configureWriter(writerBuilder, schema, options).build() + writer.withCloseable { parquetWriter -> writeRows(parquetWriter, matrix, schema, roundingMode) } + // Fail safely if the filesystem cannot atomically replace the destination. + java.nio.file.Files.move(temporary, target, java.nio.file.StandardCopyOption.ATOMIC_MOVE, + java.nio.file.StandardCopyOption.REPLACE_EXISTING) + file + } finally { java.nio.file.Files.deleteIfExists(temporary) } + } + + private static java.nio.file.Path resolveOutputTarget(java.nio.file.Path path) { + java.nio.file.Path target = path + Set visited = [] as Set + while (java.nio.file.Files.isSymbolicLink(target)) { + if (!visited.add(target.toAbsolutePath().normalize())) { + throw new java.nio.file.FileSystemLoopException(path.toString()) + } + target = target.resolveSibling(java.nio.file.Files.readSymbolicLink(target)) + } + java.nio.file.Files.exists(target) ? target.toRealPath() : target + } + + private static final String FORMAT_VERSION = '2' + private static final String DOT = '.' + + private static ExampleParquetWriter.Builder configureWriter(ExampleParquetWriter.Builder builder, MessageType schema, ParquetWriteOptions options) { + options.validate() + Set paths = schema.columns.collect { it.path.join(DOT) }.toSet() + (options.bloomFilterEnabled.keySet() + options.bloomFilterNDV.keySet()).each { String path -> + if (!paths.contains(path)) { throw new IllegalArgumentException("Unknown primitive column '$path'") } + } + builder.withRowGroupSize(options.rowGroupSize).withPageSize(options.pageSize) + .withDictionaryEncoding(options.dictionaryEncoding) + .withWriterVersion(options.writerVersion == 'V1' ? org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_1_0 : org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_2_0) + .withColumnIndexTruncateLength(options.columnIndexTruncateLength) + .withStatisticsTruncateLength(options.statisticsTruncateLength) + .withSizeStatisticsEnabled(options.sizeStatisticsEnabled) + options.bloomFilterEnabled.each { String path, Boolean enabled -> builder.withBloomFilterEnabled(path, enabled) } + options.bloomFilterNDV.each { String path, Long ndv -> builder.withBloomFilterNDV(path, ndv) } + builder } - private static Map buildExtraMeta(Matrix matrix) { - Map extraMeta = [:] - extraMeta.put(METADATA_COLUMN_TYPES, matrix.types()*.name.join(COMMA)) + private static Map buildExtraMeta(Matrix matrix, WriteSchema resolved) { + Map extraMeta = ['matrix.parquet.version': FORMAT_VERSION] + extraMeta.put(METADATA_COLUMN_TYPES, matrix.columnNames().collect { String col -> resolved.types[col].metadata.name }.join(COMMA)) if (matrix.hasIndex()) { extraMeta.put(METADATA_INDEX_COLUMNS, toJsonStringArray(matrix.indexedColumns())) } @@ -601,19 +659,20 @@ class MatrixParquetWriter { * constraints (Integer.MAX_VALUE).

* * @param matrix the matrix to write - * @param schema the Parquet MessageType schema + * @param resolved the validated physical schema and column policies * @return byte array containing Parquet data */ - private static byte[] writeBytesInternal(Matrix matrix, MessageType schema, - CompressionCodecName compressionCodec = CompressionCodecName.SNAPPY, RoundingMode roundingMode = RoundingMode.UNNECESSARY) { - def conf = new Configuration() + private static byte[] writeBytesInternal(Matrix matrix, WriteSchema resolved, + CompressionCodecName compressionCodec = CompressionCodecName.SNAPPY, RoundingMode roundingMode = RoundingMode.UNNECESSARY, ParquetWriteOptions options = new ParquetWriteOptions()) { + MessageType schema = resolved.schema + def conf = new org.apache.parquet.conf.PlainParquetConfiguration() InMemoryOutputFile outputFile = new InMemoryOutputFile() - def writer = ExampleParquetWriter.builder(outputFile) + def writerBuilder = ExampleParquetWriter.builder(outputFile) .withConf(conf) .withType(schema) .withCompressionCodec(compressionCodec) - .withExtraMetaData(buildExtraMeta(matrix)) - .build() + .withExtraMetaData(buildExtraMeta(matrix, resolved)) + def writer = configureWriter(writerBuilder, schema, options).build() writer.withCloseable { parquetWriter -> writeRows(parquetWriter, matrix, schema, roundingMode) } return outputFile.getBytes() } @@ -656,18 +715,25 @@ class MatrixParquetWriter { * * @param matrix the matrix * @param explicitDecimalMeta map of column name to [precision, scale] array for BigDecimal columns. - * @return the Parquet MessageType schema. + * @return validated physical schema and metadata policies. */ - private static MessageType buildSchema(Matrix matrix, Map explicitDecimalMeta) { - Map decimalMeta = withBigIntegerPrecision(matrix, explicitDecimalMeta) + private static WriteSchema buildSchema(Matrix matrix, Map explicitDecimalMeta, DecimalSchemaPolicy decimals = null) { + Map types = matrix.columnNames().collectEntries { String name -> + [(name): ColumnTypePolicy.resolve(name, matrix.column(name), matrix.type(name))] + } + Map decimalMeta = withBigIntegerPrecision(matrix, explicitDecimalMeta, types) + DecimalSchemaPolicy policy = decimals ?: new DecimalSchemaPolicy(explicitDecimalMeta ?: [:]) def builder = Types.buildMessage() matrix.columnNames().each { col -> def type = matrix.type(col) def meta = decimalMeta?.get(col) log.debug("Building schema for column '$col' of type $type with decimal meta: $meta") - builder.addField(buildParquetType(col, matrix.column(col), type, meta)) + builder.addField(buildParquetType(col, matrix.column(col), type, meta, false, policy.child(col), types[col])) } - return builder.named(matrix.matrixName ?: 'MatrixSchema') + Set unknown = new HashSet<>(policy.settings.keySet()) + unknown.removeAll(policy.used) + if (!unknown.empty) { throw new IllegalArgumentException("Unknown/non-decimal metadata paths: $unknown") } + new WriteSchema(builder.named(matrix.matrixName ?: 'MatrixSchema'), types) } /** @@ -675,10 +741,10 @@ class MatrixParquetWriter { * Explicit caller-supplied metadata wins after validating its scale is 0 * and that its precision is large enough to hold every value in the column. */ - private static Map withBigIntegerPrecision(Matrix matrix, Map explicitDecimalMeta) { + private static Map withBigIntegerPrecision(Matrix matrix, Map explicitDecimalMeta, Map types) { Map merged = [:] + (explicitDecimalMeta ?: [:]) matrix.columnNames().each { col -> - if (matrix.type(col) != BigInteger) { + if (types[col].encoding != BigInteger) { return } int[] existing = merged[col] @@ -687,35 +753,19 @@ class MatrixParquetWriter { throw new IllegalArgumentException( "decimalMeta['$col'] scale must be 0 for BigInteger columns but was ${existing[1]}") } - int requiredPrecision = inferBigIntegerPrecision(matrix, col) + int requiredPrecision = inferBigIntegerPrecision(matrix.column(col)) if (existing[0] < requiredPrecision) { throw new IllegalArgumentException( "decimalMeta['$col'] precision (${existing[0]}) is too small to hold the data in this " + "column (requires at least $requiredPrecision digits)") } } else { - merged[col] = [inferBigIntegerPrecision(matrix, col), 0] as int[] + merged[col] = [inferBigIntegerPrecision(matrix.column(col)), 0] as int[] } } merged } - /** - * Infers the minimum decimal precision needed to represent every BigInteger - * value in the given column as Parquet DECIMAL(precision, 0). - */ - private static int inferBigIntegerPrecision(Matrix matrix, String col) { - int maxDigits = 1 - (0.. - def value = matrix[i, col] - if (value instanceof BigInteger) { - int digits = value.abs().toString().length() - maxDigits = Math.max(maxDigits, digits) - } - } - maxDigits - } - private static int inferBigIntegerPrecision(List values) { int maxDigits = 1 values.each { value -> @@ -731,16 +781,12 @@ class MatrixParquetWriter { * Builds the Parquet schema based on the matrix, inferring decimal precision/scale if requested. * This is for backward compatibility and delegates to the explicit map builder. * @param matrix the matrix - * @param inferPrecisionAndScale if true, infers decimal precision and scale. If false, BigDecimal columns default to - * double storage (which leads to some loss of precision). - * @return the Parquet MessageType schema. + * @param inferPrecisionAndScale if true, infers decimal precision and scale. If false, BigDecimal fields require + * explicit precision/scale settings. + * @return validated physical schema and metadata policies. */ - private static MessageType buildSchema(Matrix matrix, boolean inferPrecisionAndScale = false) { - Map decimalMeta = null - if (inferPrecisionAndScale) { - decimalMeta = inferDecimalPrecisionAndScale(matrix) - } - return buildSchema(matrix, decimalMeta) + private static WriteSchema buildSchema(Matrix matrix, boolean inferPrecisionAndScale = false) { + buildSchema(matrix, [:], new DecimalSchemaPolicy([:], inferPrecisionAndScale)) } /** @@ -752,35 +798,35 @@ class MatrixParquetWriter { * @param decimalMeta optional decimal precision/scale metadata for {@link BigDecimal} columns * @return a Parquet {@link Type} describing the column */ - private static Type buildParquetType(String name, List values, Class clazz, int[] decimalMeta = null, boolean preferStructForMaps = false) { - if (isListType(clazz, values)) { + private static Type buildParquetType(String name, List values, Class clazz, int[] decimalMeta = null, boolean preferStructForMaps = false, DecimalSchemaPolicy decimals = new DecimalSchemaPolicy([:]), ColumnTypePolicy resolved = null) { + resolved = resolved ?: ColumnTypePolicy.resolve(decimals.path ?: name, values, clazz) + if (resolved.encoding == List) { List nestedValues = extractListElements(values) - Type elementType = buildParquetType(FIELD_ELEMENT, nestedValues, inferClass(nestedValues, Object), null, true) + Type elementType = buildParquetType(FIELD_ELEMENT, nestedValues, Object, null, true, decimals.child(FIELD_ELEMENT)) if (elementType == null) { elementType = buildPrimitiveType(FIELD_ELEMENT, String, null, false) } return Types.optionalList().element(elementType).named(name) } - if (isMapType(clazz, values)) { + if (resolved.encoding == Map) { if (preferStructForMaps) { - return buildStructType(name, values, clazz) + return buildStructType(name, values, clazz, decimals) } - return buildMapType(name, values) + return buildMapType(name, values, decimals) } - if (isStructType(clazz, values)) { - return buildStructType(name, values, clazz) - } - - Class effectiveClass = inferClass(values, clazz) + Class effectiveClass = resolved.encoding int[] effectiveDecimalMeta = decimalMeta + if (effectiveClass == BigDecimal) { effectiveDecimalMeta = decimals.resolve(values, effectiveDecimalMeta) } + if (effectiveClass == BigInteger && decimals.settings.containsKey(decimals.path)) { decimals.used.add(decimals.path) } if (effectiveClass == BigInteger && effectiveDecimalMeta == null) { effectiveDecimalMeta = [inferBigIntegerPrecision(values), 0] as int[] } return buildPrimitiveType(name, effectiveClass, effectiveDecimalMeta, false) } + @SuppressWarnings(['AbcMetric', 'CyclomaticComplexity']) // Exhaustive physical/logical type dispatch. private static PrimitiveType buildPrimitiveType(String name, Class clazz, int[] decimalMeta = null, boolean required = false) { if (clazz == BigInteger) { if (decimalMeta == null || decimalMeta.length != DECIMAL_META_LENGTH || decimalMeta[0] <= 0 || decimalMeta[1] != 0) { @@ -804,15 +850,31 @@ class MatrixParquetWriter { .as(LogicalTypeAnnotation.decimalType(scale, precision)) return builder.named(name) } - // fallback to double (discouraged, but allowed for backward compat) - def fallbackBuilder = required ? Types.required(PrimitiveTypeName.DOUBLE) : Types.optional(PrimitiveTypeName.DOUBLE) - return fallbackBuilder.named(name) + throw new IllegalArgumentException("BigDecimal field '$name' requires precision/scale") + } + + if (clazz == UUID) { + def uuidBuilder = required ? Types.required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) : Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) + return uuidBuilder.length(16).as(LogicalTypeAnnotation.uuidType()).named(name) } def primitive def logical = null switch (clazz) { + case Byte, Short -> { + primitive = PrimitiveTypeName.INT32 + logical = LogicalTypeAnnotation.intType(clazz == Byte ? 8 : 16, true) + } + case byte[] -> primitive = PrimitiveTypeName.BINARY + case String, Character -> { + primitive = PrimitiveTypeName.BINARY + logical = LogicalTypeAnnotation.stringType() + } + case LocalTime -> { + primitive = PrimitiveTypeName.INT64 + logical = LogicalTypeAnnotation.timeType(false, LogicalTypeAnnotation.TimeUnit.NANOS) + } case Integer, int -> primitive = PrimitiveTypeName.INT32 case Long, long -> primitive = PrimitiveTypeName.INT64 case Float, float -> primitive = PrimitiveTypeName.FLOAT @@ -822,7 +884,7 @@ class MatrixParquetWriter { primitive = PrimitiveTypeName.INT32 logical = LogicalTypeAnnotation.dateType() } - case LocalDateTime, Timestamp -> { + case LocalDateTime, Timestamp, Instant -> { primitive = PrimitiveTypeName.INT64 logical = LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MICROS) } @@ -834,7 +896,7 @@ class MatrixParquetWriter { primitive = PrimitiveTypeName.INT64 logical = LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS) } - default -> primitive = PrimitiveTypeName.BINARY + default -> throw new IllegalArgumentException("Unsupported primitive $clazz for field $name") } def builder = required ? Types.required(primitive) : Types.optional(primitive) @@ -844,33 +906,33 @@ class MatrixParquetWriter { return builder.named(name) } - private static Type buildMapType(String name, List values) { + private static Type buildMapType(String name, List values, DecimalSchemaPolicy decimals) { def mapBuilder = Types.optionalMap() List maps = values.findAll { it instanceof Map } as List + if (maps.any { it.containsKey(null) }) { throw new IllegalArgumentException("Map field '$name' cannot contain null keys") } List keySamples = maps.collectMany { it?.keySet() ?: [] } as List - Object keySample = findFirstNonNull(keySamples) - Class keyClass = inferClass(keySamples, keySample?.class ?: String) - int[] keyDecimalMeta = keyClass == BigInteger ? [inferBigIntegerPrecision(keySamples), 0] as int[] : null + Class keyClass = ColumnTypePolicy.resolve(decimals.child(FIELD_KEY).path, keySamples, Object).encoding + int[] keyDecimalMeta = keyClass == BigInteger ? [inferBigIntegerPrecision(keySamples), 0] as int[] : keyClass == BigDecimal ? decimals.child(FIELD_KEY).resolve(keySamples, null) : null PrimitiveType keyType = buildPrimitiveType(FIELD_KEY, keyClass ?: String, keyDecimalMeta, true) mapBuilder.key(keyType) List valueSamples = maps.collectMany { it?.values() ?: [] } as List Set> valueClasses = valueSamples.findAll { it != null }*.class.toSet() if (valueClasses.size() > 1) { - return buildStructType(name, values, Map) + return buildStructType(name, values, Map, decimals) } Type valueType if (valueSamples.isEmpty()) { valueType = buildPrimitiveType(FIELD_VALUE, String, null, false) } else { - valueType = buildParquetType(FIELD_VALUE, valueSamples, inferClass(valueSamples, Object), null) + valueType = buildParquetType(FIELD_VALUE, valueSamples, Object, null, false, decimals.child(FIELD_VALUE)) } mapBuilder.value(valueType) return mapBuilder.named(name) } - private static Type buildStructType(String name, List values, Class declaredType) { + private static Type buildStructType(String name, List values, Class declaredType, DecimalSchemaPolicy decimals) { Object sample = findFirstNonNull(values) if (sample == null) { return Types.optionalGroup().named(name) @@ -884,7 +946,10 @@ class MatrixParquetWriter { Set keys = new LinkedHashSet<>() values.each { val -> if (val instanceof Map mapVal) { - mapVal.keySet().each { key -> keys.add(String.valueOf(key)) } + mapVal.keySet().each { key -> + if (!(key instanceof String)) { throw new IllegalArgumentException("Struct field '$name' requires String keys; got ${key?.getClass()?.name}") } + keys.add(key) + } } } keys.each { key -> @@ -897,7 +962,7 @@ class MatrixParquetWriter { } } fieldValues[key] = collected - fieldTypes[key] = inferClass(collected, Object) + fieldTypes[key] = Object } } else { List descriptors = ParquetValueWriter.getPropertyDescriptors(sample.class) @@ -920,7 +985,7 @@ class MatrixParquetWriter { } fieldValues.each { fieldName, collected -> - Type fieldType = buildParquetType(fieldName, collected, fieldTypes[fieldName] ?: Object, null) + Type fieldType = buildParquetType(fieldName, collected, fieldTypes[fieldName] ?: Object, null, false, decimals.child(fieldName)) groupBuilder.addField(fieldType) } return groupBuilder.named(name) @@ -936,40 +1001,6 @@ class MatrixParquetWriter { return elements } - private static boolean isListType(Class declaredType, List values) { - if (declaredType != null && Collection.isAssignableFrom(declaredType)) { - return true - } - return values.any { it instanceof Collection } - } - - private static boolean isMapType(Class declaredType, List values) { - if (declaredType != null && Map.isAssignableFrom(declaredType)) { - return true - } - return values.any { it instanceof Map } - } - - private static boolean isStructType(Class declaredType, List values) { - if (isMapType(declaredType, values)) { - return true - } - Class effective = inferClass(values, declaredType) - return effective != null && !isPrimitiveLike(effective) && !Collection.isAssignableFrom(effective) - } - - private static Class inferClass(List values, Class declaredType) { - Object sample = findFirstNonNull(values) - if (sample != null) { - Class clazz = sample.class - return clazz.isPrimitive() ? convertPrimitive(clazz) : clazz - } - if (declaredType != null && declaredType != Object) { - return declaredType.isPrimitive() ? convertPrimitive(declaredType) : declaredType - } - return Object - } - private static Object findFirstNonNull(List values) { for (def val : values) { if (val != null) { @@ -979,96 +1010,18 @@ class MatrixParquetWriter { return null } - private static Class convertPrimitive(Class clazz) { - if (!clazz.isPrimitive()) { - return clazz - } - switch (clazz) { - case int -> Integer - case long -> Long - case double -> Double - case float -> Float - case boolean -> Boolean - case byte -> Byte - case short -> Short - case char -> Character - default -> clazz - } - } - - private static boolean isPrimitiveLike(Class clazz) { - if (clazz == null) { - return false - } - if (clazz.isPrimitive()) { - return true - } - if (Number.isAssignableFrom(clazz)) { - return true - } - if (CharSequence.isAssignableFrom(clazz)) { - return true - } - if (Boolean.isAssignableFrom(clazz)) { - return true - } - if (Date.isAssignableFrom(clazz)) { - return true - } - return clazz in [LocalDate, LocalDateTime, Timestamp, Time] + private static int minBytesForPrecision(int precision) { + int bitsNeeded = BigInteger.TEN.pow(precision).subtract(BigInteger.ONE).bitLength() + 1 + Math.ceil(bitsNeeded / 8d) as int } - /** - * Infers the required maximum precision and scale for all BigDecimal columns in the matrix. - * It performs the calculation in a single pass: - * 1. It tracks the running maximum scale (S). - * 2. It tracks the running maximum number of digits to the LEFT of the decimal (L). - * 3. The final precision P = L + S. - * This guarantees that the resulting schema is large enough to contain all values. - * @param matrix the matrix - * @return a Map of column name to [precision, scale] array. - */ - private static Map inferDecimalPrecisionAndScale(Matrix matrix) { - Map result = [:] // columnName -> [precision, scale] + private static final class WriteSchema { + final MessageType schema + final Map types - matrix.columnNames().each { col -> - if (matrix.type(col) == BigDecimal) { - int maxScale = 0 - int maxLeftDigits = 0 // Max digits to the left of the decimal - - (0.. - def value = matrix[i, col] - if (value instanceof BigDecimal) { - int valScale = value.scale() - maxScale = Math.max(maxScale, valScale) - - // Calculate digits to the left of the decimal - // precision() - scale() works for most numbers (e.g., 123.45 -> p=5, s=2 -> 3) - // but fails for numbers < 1 (e.g., 0.45 -> p=2, s=2 -> 0) - // We must ensure at least one digit (for the '0') - int leftDigits = Math.max(1, value.precision() - value.scale()) - maxLeftDigits = Math.max(maxLeftDigits, leftDigits) - } - } - - // Total precision = max digits left + max digits right (max scale) - int maxPrecision = maxLeftDigits + maxScale - - // Ensure precision is at least 1 (e.g., for a column of all 0s) - maxPrecision = Math.max(1, maxPrecision) - - result[col] = [maxPrecision, maxScale] as int[] - } + WriteSchema(MessageType schema, Map types) { + this.schema = schema + this.types = types } - log.debug("Inferred decimal precision/scale: $result") - return result - } - - - private static int minBytesForPrecision(int precision) { - // According to the Parquet spec: enough bits to hold [-(10^precision - 1), 10^precision - 1] - // in two's complement, i.e. the bit length of the largest magnitude plus a sign bit. - int bitsNeeded = BigInteger.TEN.pow(precision).subtract(BigInteger.ONE).bitLength() + 1 - return Math.ceil(bitsNeeded / 8d) as int } } diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetBatches.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetBatches.groovy new file mode 100644 index 000000000..99505b50b --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetBatches.groovy @@ -0,0 +1,222 @@ +package se.alipsa.matrix.parquet + +import org.apache.parquet.conf.PlainParquetConfiguration +import org.apache.parquet.filter2.compat.FilterCompat +import org.apache.parquet.hadoop.ParquetFileReader +import org.apache.parquet.hadoop.metadata.ParquetMetadata +import org.apache.parquet.io.ColumnIOFactory +import org.apache.parquet.io.InputFile +import org.apache.parquet.io.RecordReader +import org.apache.parquet.schema.MessageType +import org.apache.parquet.schema.Type + +import se.alipsa.matrix.core.Matrix + +/** Closeable row-group iterator. A row group may be split with maxBatchRows; close on early exit. */ +final class ParquetBatches implements MatrixBatchIterator { + private static final String COMMA = ',' + private static final int KEEP_EMPTY = -1 + private final ParquetFileReader reader + private final ParquetReadOptions options + private final MessageType requested + private final List outputNames + private final List types + private final List selected + private final Map groupIndexes + private final ParquetPredicate residual + private final FilterCompat.Filter filter + private final String name + private final String sourcePath + private final String indexMetadata + private final boolean binaryAsString + private ColumnMaterializer materializer + private RecordReader records + private org.apache.parquet.column.page.PageReadStore pages + private long rowsRemaining + private long remaining + private int groupPosition + private boolean closed + private Matrix pending + + /** Opens an input and parses its footer exactly once. */ + @SuppressWarnings('AbcMetric') // Validates and snapshots all footer-dependent state before data I/O. + ParquetBatches(InputFile input, ParquetReadOptions options, String fallbackName, ParquetMetadata snapshot = null, String sourcePath = null) { + options.validate() + this.options = options + this.sourcePath = sourcePath + if (options.sourceFileColumn != null && sourcePath == null) { throw new IllegalArgumentException('sourceFileColumn requires a local file') } + def conf = new PlainParquetConfiguration() + def baseOptions = org.apache.parquet.ParquetReadOptions.builder(conf).build() + ParquetMetadata footer = snapshot + if (footer == null) { + input.newStream().withCloseable { stream -> footer = ParquetFileReader.readFooter(input, baseOptions, stream) } + } + MessageType full = footer.fileMetaData.schema + if (full.fields.empty || full.fields.any { it.repetition == Type.Repetition.REPEATED }) { + throw new IllegalArgumentException('Parquet requires nonempty top-level fields; wrap repeated fields in LIST') + } + def metadata = footer.fileMetaData.keyValueMetaData + name = options.matrixName ?: fallbackName ?: full.name + indexMetadata = metadata[MatrixParquetReader.METADATA_INDEX_COLUMNS] + String typeString = metadata[MatrixParquetReader.METADATA_COLUMN_TYPES] + boolean legacy = typeString != null && !metadata.containsKey('matrix.parquet.version') + + binaryAsString = options.binaryAsString != null ? options.binaryAsString : legacy + outputNames = options.columns == null ? full.fields*.name : new ArrayList<>(options.columns) + outputNames.each { String column -> + if (!full.containsField(column)) { throw new IllegalArgumentException("Unknown column '$column'") } + } + residual = options.filter?.bind(full, options.zoneId) + Set needed = new LinkedHashSet<>(outputNames) + if (options.filter != null) { needed.addAll(options.filter.columns()) } + validateLegacyBinary(full, typeString, needed, legacy) + if (options.sourceFileColumn != null && full.containsField(options.sourceFileColumn)) { throw new IllegalArgumentException('sourceFileColumn collides with file schema') } + requested = new MessageType(full.name, full.fields.findAll { needed.contains(it.name) }) + List inferred = MatrixParquetReader.extractFieldTypes(full) + List declared = typeString == null ? [] : typeString.split(COMMA, KEEP_EMPTY).toList() + types = requested.fields.collect { Type field -> + int index = full.getFieldIndex(field.name) + Class explicit = declared.size() == full.fieldCount ? ColumnTypePolicy.METADATA_TYPES[declared[index]] : null + explicit = compatibleMetadata(explicit, field) ? explicit : null + explicit ?: (binaryAsString && field.primitive && field.asPrimitiveType().logicalTypeAnnotation == null && + field.asPrimitiveType().primitiveTypeName == org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY ? String : inferred[index]) + } + selected = options.rowGroups == null ? (0..(options.rowGroups) + selected.each { Integer id -> + if (id == null || id < 0 || id >= footer.blocks.size()) { throw new IllegalArgumentException("Invalid row-group ID $id") } + } + def pushed = residual?.pushdown(full) + filter = pushed == null ? FilterCompat.NOOP : FilterCompat.get(pushed) + def readOptions = org.apache.parquet.ParquetReadOptions.builder(conf).withRecordFilter(filter) + .useColumnIndexFilter(options.useColumnIndexFilter) + .useStatsFilter(options.useRowGroupFilters).useDictionaryFilter(options.useRowGroupFilters) + .useBloomFilter(options.useRowGroupFilters).build() + def stream = input.newStream() + try { + reader = ParquetFileReader.open(input, footer, readOptions, stream) + reader.setRequestedSchema(requested) + } catch (Exception failure) { + stream.close() + throw failure + } + groupIndexes = [:] + reader.rowGroups.eachWithIndex { block, int index -> groupIndexes[footer.blocks.indexOf(block)] = index } + remaining = options.limit == null ? Long.MAX_VALUE : options.limit + } + + private static void validateLegacyBinary(MessageType full, String typeString, Set needed, boolean legacy) { + List legacyTypes = typeString == null ? [] : typeString.split(COMMA, KEEP_EMPTY).toList() + if (legacy && legacyTypes.size() == full.fieldCount && full.fields.any { field -> needed.contains(field.name) && legacyTypes[full.getFieldIndex(field.name)] == '[B' }) { + throw new IllegalArgumentException('Unrecoverable legacy Matrix byte[]: original binary payload was not preserved') + } + } + + private static boolean compatibleMetadata(Class expected, Type field) { + if (expected == null) { return false } + if (!field.primitive) { + boolean list = field.logicalTypeAnnotation instanceof org.apache.parquet.schema.LogicalTypeAnnotation.ListLogicalTypeAnnotation + return list ? expected == List : expected == Map + } + def primitive = field.asPrimitiveType() + def logical = primitive.logicalTypeAnnotation + if (logical instanceof org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { return expected == BigDecimal || (expected == BigInteger && logical.scale == 0) } + if (logical instanceof org.apache.parquet.schema.LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { return expected in [Date, java.sql.Timestamp, java.time.LocalDateTime, java.time.Instant] } + if (logical instanceof org.apache.parquet.schema.LogicalTypeAnnotation.TimeLogicalTypeAnnotation) { return expected in [java.sql.Time, java.time.LocalTime, Date] } + if (logical instanceof org.apache.parquet.schema.LogicalTypeAnnotation.DateLogicalTypeAnnotation) { return expected in [java.sql.Date, java.time.LocalDate, Date] } + if (logical instanceof org.apache.parquet.schema.LogicalTypeAnnotation.UUIDLogicalTypeAnnotation) { return expected == UUID } + if (MatrixParquetReader.isTextLogicalType(logical)) { return expected == String } + compatiblePhysicalMetadata(expected, primitive.primitiveTypeName) + } + + private static boolean compatiblePhysicalMetadata(Class expected, + org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName physical) { + switch (physical) { + case org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32 -> expected in [Byte, Short, Integer] + case org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64 -> expected == Long + case org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT -> expected == Float + case org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE -> expected == Double + case org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BOOLEAN -> expected == Boolean + case org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY -> expected in [String, byte[], BigDecimal] + case org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY -> expected == byte[] + default -> false + } + } + + /** Returns an empty Matrix with the selected names and types. */ + Matrix emptyMatrix() { + List empty = requested.fields.collect { [] } + build(empty) + } + + private Matrix build(List data) { + List indexes = outputNames.collect { requested.getFieldIndex(it) } + List copied = indexes.collect { data[it] } + Matrix result = Matrix.builder(name).columnNames(outputNames).types(indexes.collect { types[it] }) + .columns(copied).build() + MatrixParquetReader.restoreIndex(result, indexMetadata) + if (options.sourceFileColumn != null) { result.addColumn(options.sourceFileColumn, String, [sourcePath] * result.rowCount()) } + result + } + + @Override + boolean hasNext() { + MatrixParquetReader.withReadZone(options.zoneId) { advance() } + } + + private boolean advance() { + if (pending != null) { return true } + if (closed) { return false } + try { + while (remaining > 0) { + if (rowsRemaining == 0) { + pages?.close() + pages = null + if (groupPosition >= selected.size()) { close(); return false } + Integer index = groupIndexes[selected[groupPosition++]] + if (index == null) { continue } + pages = reader.readFilteredRowGroup(index) + if (pages == null) { continue } + rowsRemaining = pages.rowCount + materializer = new ColumnMaterializer(requested, types, binaryAsString, residual) + def columnIO = new ColumnIOFactory(reader.fileMetaData.createdBy).getColumnIO(requested, reader.fileMetaData.schema, true) + records = columnIO.getRecordReader(pages, materializer, filter) + } + int cap = (int) Math.min(remaining, options.maxBatchRows ?: Integer.MAX_VALUE) + while (rowsRemaining > 0 && materializer.columns[0].size() < cap) { + if (Thread.currentThread().isInterrupted()) { throw new InterruptedIOException('Parquet decoding cancelled') } + try { records.read() } catch (Exception failure) { materializer.skipCurrentRecord(); throw failure } + rowsRemaining-- + } + if (!materializer.columns[0].empty) { + int count = materializer.columns[0].size() + pending = build(materializer.columns) + materializer.columns.each { it.clear() } + remaining -= count + return true + } + } + close() + false + } catch (Exception failure) { + close() + throw failure + } + } + + @Override + Matrix next() { + if (!hasNext()) { throw new NoSuchElementException() } + Matrix result = pending + pending = null + result + } + + @Override + void close() { + if (!closed) { + closed = true + pending = null + try { pages?.close() } finally { reader.close() } + } + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetFormatProvider.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetFormatProvider.groovy index a35d731e8..3458092c8 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetFormatProvider.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetFormatProvider.groovy @@ -33,49 +33,17 @@ class ParquetFormatProvider extends AbstractFormatProvider { @Override Matrix read(File file, Map options) { - ParquetReadOptions readOptions = ParquetReadOptions.fromMap(options) - if (readOptions.matrixName != null && readOptions.zoneId != null) { - return MatrixParquetReader.read(file, readOptions.matrixName, readOptions.zoneId) - } - if (readOptions.matrixName != null) { - return MatrixParquetReader.read(file, readOptions.matrixName) - } - if (readOptions.zoneId != null) { - return MatrixParquetReader.read(file, readOptions.zoneId) - } - MatrixParquetReader.read(file) + MatrixParquetReader.builder(ParquetReadOptions.fromMap(options)).read(file) } @Override Matrix read(URL url, Map options) { - ParquetReadOptions readOptions = ParquetReadOptions.fromMap(options) - if (readOptions.matrixName != null && readOptions.zoneId != null) { - return MatrixParquetReader.read(url, readOptions.matrixName, readOptions.zoneId) - } - if (readOptions.matrixName != null) { - url.openStream().withCloseable { InputStream is -> - MatrixParquetReader.read(is, readOptions.matrixName) - } - } else if (readOptions.zoneId != null) { - MatrixParquetReader.read(url, readOptions.zoneId) - } else { - MatrixParquetReader.read(url) - } + MatrixParquetReader.builder(ParquetReadOptions.fromMap(options)).read(url) } @Override - Matrix read(InputStream is, Map options) { - ParquetReadOptions readOptions = ParquetReadOptions.fromMap(options) - if (readOptions.matrixName != null && readOptions.zoneId != null) { - return MatrixParquetReader.read(is, readOptions.matrixName, readOptions.zoneId) - } - if (readOptions.matrixName != null) { - return MatrixParquetReader.read(is, readOptions.matrixName) - } - if (readOptions.zoneId != null) { - return MatrixParquetReader.read(is, readOptions.zoneId) - } - MatrixParquetReader.read(is) + Matrix read(InputStream stream, Map options) { + MatrixParquetReader.builder(ParquetReadOptions.fromMap(options)).read(stream) } @Override diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetPredicate.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetPredicate.groovy new file mode 100644 index 000000000..bd4f46ea5 --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetPredicate.groovy @@ -0,0 +1,210 @@ +package se.alipsa.matrix.parquet + +import groovy.transform.PackageScope + +import org.apache.parquet.filter2.predicate.FilterApi +import org.apache.parquet.filter2.predicate.FilterPredicate +import org.apache.parquet.io.api.Binary +import org.apache.parquet.schema.LogicalTypeAnnotation +import org.apache.parquet.schema.MessageType +import org.apache.parquet.schema.PrimitiveType + +import java.time.Instant +import java.time.LocalDate +import java.time.LocalDateTime +import java.time.LocalTime +import java.time.ZoneId + +/** Immutable logical predicate; values are converted against the file schema before reading. */ +final class ParquetPredicate { + private static final String IS_NOT_NULL = 'isNotNull' + + private static final String KEY_EQ = 'eq' + private static final String KEY_NE = 'ne' + private static final String KEY_LT = 'lt' + private static final String KEY_LE = 'le' + private static final String KEY_GT = 'gt' + private static final String KEY_GE = 'ge' + + final String operation + final String column + private final Object value + final List children + + @PackageScope + ParquetPredicate(String operation, String column, Object value, List children = []) { + if (column == null && (children.empty || children.any { it == null })) { throw new IllegalArgumentException('Predicate children cannot be null') } + if (column != null && (column.empty || (!(operation in ['isNull', IS_NOT_NULL]) && value == null))) { throw new IllegalArgumentException('Use explicit null predicates for null comparisons') } + this.operation = operation + this.column = column + this.value = value instanceof byte[] ? ((byte[]) value).clone() : + value instanceof Number && !(value instanceof Double) && !(value instanceof Float) ? new BigDecimal(value.toString()) : value + this.children = Collections.unmodifiableList(new ArrayList<>(children)) + } + + /** Returns the top-level columns required by this predicate. */ + Set columns() { + column == null ? children.collectMany { it.columns() }.toSet() : [column] as Set + } + + /** Evaluates the exact logical predicate; comparisons with null are false except null tests. */ + boolean test(Map row) { + switch (operation) { + case 'and' -> children.every { it.test(row) } + case 'or' -> children.any { it.test(row) } + case 'not' -> !children[0].test(row) + case 'isNull' -> row[column] == null + case IS_NOT_NULL -> row[column] != null + default -> compare(row[column]) + } + } + + private boolean compare(Object actual) { + if (actual == null || value == null) { return false } + int comparison + if (actual instanceof Number && value instanceof Number) { + if (Double.isNaN(actual.doubleValue()) || Double.isNaN(((Number) value).doubleValue())) { + return operation == KEY_NE + } + boolean actualInfinite = (actual instanceof Double || actual instanceof Float) && Double.isInfinite(actual.doubleValue()) + boolean valueInfinite = (value instanceof Double || value instanceof Float) && Double.isInfinite(((Number) value).doubleValue()) + comparison = actualInfinite && valueInfinite ? Double.compare(actual.doubleValue(), ((Number) value).doubleValue()) : + actualInfinite ? (actual.doubleValue() > 0 ? 1 : -1) : valueInfinite ? (((Number) value).doubleValue() > 0 ? -1 : 1) : + new BigDecimal(actual.toString()) <=> new BigDecimal(value.toString()) + } else if (actual instanceof String && value instanceof String) { + comparison = Arrays.compareUnsigned(actual.getBytes(java.nio.charset.StandardCharsets.UTF_8), + ((String) value).getBytes(java.nio.charset.StandardCharsets.UTF_8)) + } else if (actual instanceof byte[] && value instanceof byte[]) { + comparison = Arrays.compareUnsigned((byte[]) actual, (byte[]) value) + } else { + comparison = ((Comparable) actual) <=> value + } + switch (operation) { + case KEY_EQ -> comparison == 0 + case KEY_NE -> comparison != 0 + case KEY_LT -> comparison < 0 + case KEY_LE -> comparison <= 0 + case KEY_GT -> comparison > 0 + case KEY_GE -> comparison >= 0 + default -> throw new IllegalArgumentException("Unknown predicate $operation") + } + } + + // Only exact physical conversions are pushed down; unsafe expressions remain wholly residual. + @PackageScope + FilterPredicate pushdown(MessageType schema) { + if (column == null) { + List predicates = children*.pushdown(schema) + if (predicates.any { it == null }) { return null } + return switch (operation) { + case 'and' -> FilterApi.and(predicates[0], predicates[1]) + case 'or' -> FilterApi.or(predicates[0], predicates[1]) + default -> null + } + } + TypeCheck.validate(this, schema) + PrimitiveType field = schema.getType(column).asPrimitiveType() + def logical = field.logicalTypeAnnotation + if (operation in [KEY_NE, IS_NOT_NULL]) { return null } + if (logical instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation && + field.primitiveTypeName in [PrimitiveType.PrimitiveTypeName.BINARY, PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY]) { return null } + if (!(field.primitiveTypeName in [PrimitiveType.PrimitiveTypeName.INT32, PrimitiveType.PrimitiveTypeName.INT64, PrimitiveType.PrimitiveTypeName.BINARY])) { return null } + Object physical = physicalValue(field) + def col + switch (field.primitiveTypeName) { + case PrimitiveType.PrimitiveTypeName.INT32 -> { + col = FilterApi.intColumn(column) + if (physical != null) { physical = new BigDecimal(physical.toString()).intValueExact() } + } + case PrimitiveType.PrimitiveTypeName.INT64 -> { + col = FilterApi.longColumn(column) + if (physical != null) { physical = new BigDecimal(physical.toString()).longValueExact() } + } + case PrimitiveType.PrimitiveTypeName.BINARY -> { + col = FilterApi.binaryColumn(column) + physical = physical instanceof String ? Binary.fromString((String) physical) : physical instanceof byte[] ? Binary.fromConstantByteArray((byte[]) physical) : null + } + default -> col = null + } + buildPhysical(operation, col, (Comparable) physical) + } + + @groovy.transform.CompileDynamic + private static FilterPredicate buildPhysical(String operation, col, Comparable physical) { + switch (operation) { + case KEY_EQ, 'isNull' -> FilterApi.eq(col, physical) + case KEY_LT -> FilterApi.lt(col, physical) + case KEY_LE -> FilterApi.ltEq(col, physical) + case KEY_GT -> FilterApi.gt(col, physical) + case KEY_GE -> FilterApi.gtEq(col, physical) + default -> null + } + } + + @PackageScope + ParquetPredicate bind(MessageType schema, ZoneId zone) { + if (column == null) { return new ParquetPredicate(operation, null, null, children*.bind(schema, zone)) } + TypeCheck.validate(this, schema) + PrimitiveType field = schema.getType(column).asPrimitiveType() + Object normalized = value + if (field.logicalTypeAnnotation instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation && + ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) field.logicalTypeAnnotation).isAdjustedToUTC() && value instanceof LocalDateTime) { + normalized = value.atZone(zone ?: ZoneId.systemDefault()).toInstant() + } + ParquetPredicate bound = new ParquetPredicate(operation, column, normalized) + bound.physicalValue(field) + bound + } + + private Object physicalValue(PrimitiveType field) { + if (value == null) { return null } + def logical = field.logicalTypeAnnotation + if (logical instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { + BigDecimal decimal = new BigDecimal(value.toString()).setScale(logical.scale, java.math.RoundingMode.UNNECESSARY) + if (decimal.precision() > logical.precision) { throw new IllegalArgumentException("Decimal predicate overflows '${field.name}'") } + return decimal.unscaledValue() + } + if (value instanceof LocalDate) { return Math.toIntExact(value.toEpochDay()) } + if (logical instanceof LogicalTypeAnnotation.TimeLogicalTypeAnnotation) { + long nanos = ((LocalTime) value).toNanoOfDay() + long divisor = logical.unit == LogicalTypeAnnotation.TimeUnit.MILLIS ? 1_000_000L : logical.unit == LogicalTypeAnnotation.TimeUnit.MICROS ? 1000L : 1L + if (nanos % divisor != 0) { throw new IllegalArgumentException("Lossy TIME predicate '${field.name}'") } + return nanos.intdiv(divisor) + } + if (logical instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { + Instant instant = value instanceof Instant ? (Instant) value : ((LocalDateTime) value).toInstant(java.time.ZoneOffset.UTC) + long divisor = logical.unit == LogicalTypeAnnotation.TimeUnit.MILLIS ? 1_000_000L : logical.unit == LogicalTypeAnnotation.TimeUnit.MICROS ? 1000L : 1L + if (instant.nano % divisor != 0) { throw new IllegalArgumentException("Lossy TIMESTAMP predicate '${field.name}'") } + return Math.addExact(Math.multiplyExact(instant.epochSecond, 1_000_000_000L.intdiv(divisor)), instant.nano.intdiv(divisor)) + } + if (field.primitiveTypeName == PrimitiveType.PrimitiveTypeName.INT32) { return new BigDecimal(value.toString()).intValueExact() } + if (field.primitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) { return new BigDecimal(value.toString()).longValueExact() } + value + } + + private static class TypeCheck { + static void validate(ParquetPredicate predicate, MessageType schema) { + if (!schema.containsField(predicate.column)) { + throw new IllegalArgumentException("Unknown predicate column '${predicate.column}'") + } + def field = schema.getType(predicate.column) + if (!field.primitive || field.repetition == org.apache.parquet.schema.Type.Repetition.REPEATED) { + throw new IllegalArgumentException("Repeated/nested predicate '${predicate.column}' is unsupported") + } + if (predicate.operation in ['isNull', IS_NOT_NULL]) { return } + Class inferred = MatrixParquetReader.extractFieldTypes(new MessageType('predicate', [field]))[0] + Object value = predicate.value + boolean compatible = Number.isAssignableFrom(inferred) ? value instanceof Number : + inferred == java.sql.Time ? value instanceof LocalTime : + inferred == LocalDateTime ? (value instanceof LocalDateTime || + (value instanceof Instant && ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) field.logicalTypeAnnotation).isAdjustedToUTC())) : inferred.isInstance(value) + if (!compatible) { throw new IllegalArgumentException("Incompatible predicate value for '${predicate.column}': expected $inferred") } + } + } + + @PackageScope + void validate(MessageType schema) { + if (column != null) { TypeCheck.validate(this, schema) } + children.each { it.validate(schema) } + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetReadOptions.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetReadOptions.groovy index 13e2a50d6..222ac9cd9 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetReadOptions.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetReadOptions.groovy @@ -9,6 +9,98 @@ import java.time.ZoneId * Typed options for Parquet read operations via the SPI. */ class ParquetReadOptions { + private static final String KEY_REMOTE_READ_MODE = 'remoteReadMode' + private static final String DOWNLOAD = 'DOWNLOAD' + private static final String KEY_CONNECT_TIMEOUT_MILLIS = 'connectTimeoutMillis' + private static final String KEY_READ_TIMEOUT_MILLIS = 'readTimeoutMillis' + private static final String KEY_RANGE_CACHE_BYTES = 'rangeCacheBytes' + private static final String KEY_SOURCE_FILE_COLUMN = 'sourceFileColumn' + private static final String KEY_PARALLELISM = 'parallelism' + private static final String KEY_LIMIT = 'limit' + private static final String KEY_ROW_GROUPS_BY_FILE = 'rowGroupsByFile' + private static final String KEY_ROW_GROUPS = 'rowGroups' + private static final String KEY_MAX_BATCH_ROWS = 'maxBatchRows' + + private static final String KEY_COLUMNS = 'columns' + private static final String KEY_FILTER = 'filter' + private static final String KEY_BINARY_AS_STRING = 'binaryAsString' + private static final String KEY_USE_COLUMN_INDEX_FILTER = 'useColumnIndexFilter' + + String remoteReadMode = DOWNLOAD + int connectTimeoutMillis = 10000 + int readTimeoutMillis = 60000 + int rangeCacheBytes = 8388608 + + @groovy.transform.PackageScope + String hubToken = null + String sourceFileColumn = null + int parallelism = 1 + + @groovy.transform.PackageScope + boolean useRowGroupFilters = true + + List columns = null + ParquetPredicate filter = null + Boolean binaryAsString = null + boolean useColumnIndexFilter = true + Long limit = null + List rowGroups = null + Map> rowGroupsByFile = [:] + Integer maxBatchRows = null + + /** Validates bounds and duplicate selectors before opening data pages. */ + void validate() { + if (!(remoteReadMode in [DOWNLOAD, 'AUTO', 'RANGE'])) { throw new IllegalArgumentException('remoteReadMode must be DOWNLOAD, AUTO or RANGE') } + if (connectTimeoutMillis <= 0 || readTimeoutMillis <= 0 || rangeCacheBytes <= 0) { throw new IllegalArgumentException('HTTP timeouts and rangeCacheBytes must be positive') } + + if (parallelism <= 0) { throw new IllegalArgumentException('parallelism must be positive') } + if (limit != null && limit < 0) { throw new IllegalArgumentException('limit must be nonnegative') } + if (maxBatchRows != null && maxBatchRows <= 0) { throw new IllegalArgumentException('maxBatchRows must be positive') } + if (columns != null && (columns.empty || columns.toSet().size() != columns.size())) { throw new IllegalArgumentException('columns must be nonempty and unique') } + if (rowGroups != null && rowGroups.toSet().size() != rowGroups.size()) { throw new IllegalArgumentException('rowGroups must be unique') } + } + + /** Transport: DOWNLOAD, AUTO or RANGE; default DOWNLOAD. */ + ParquetReadOptions remoteReadMode(String value) { this.remoteReadMode = value; this } + + /** HTTP connection timeout; default 10000. */ + ParquetReadOptions connectTimeoutMillis(int value) { this.connectTimeoutMillis = value; this } + + /** HTTP read timeout; default 60000. */ + ParquetReadOptions readTimeoutMillis(int value) { this.readTimeoutMillis = value; this } + + /** Maximum HTTP range cache bytes; default 8388608. */ + ParquetReadOptions rangeCacheBytes(int value) { this.rangeCacheBytes = value; this } + + /** Optional local source path column; default null. */ + ParquetReadOptions sourceFileColumn(String value) { this.sourceFileColumn = value; this } + + /** Bounded dataset decoder workers; default 1. */ + ParquetReadOptions parallelism(int value) { this.parallelism = value; this } + + /** Ordered output columns; default null. */ + ParquetReadOptions columns(List value) { this.columns = value; this } + + /** Exact logical row filter; default null. */ + ParquetReadOptions filter(ParquetPredicate value) { this.filter = value; this } + + /** Unannotated BINARY override; default null. */ + ParquetReadOptions binaryAsString(Boolean value) { this.binaryAsString = value; this } + + /** Enable page-index pruning; default true. */ + ParquetReadOptions useColumnIndexFilter(boolean value) { this.useColumnIndexFilter = value; this } + + /** Maximum accepted rows; default null. */ + ParquetReadOptions limit(Long value) { this.limit = value; this } + + /** Ordered original footer IDs; default null. */ + ParquetReadOptions rowGroups(List value) { this.rowGroups = value; this } + + /** Per-file original footer group IDs, keyed by canonical local path; default empty/all. */ + ParquetReadOptions rowGroupsByFile(Map> value) { rowGroupsByFile = value; this } + + /** Maximum rows per batch; default null. */ + ParquetReadOptions maxBatchRows(Integer value) { this.maxBatchRows = value; this } String matrixName = null ZoneId zoneId = null @@ -36,12 +128,20 @@ class ParquetReadOptions { if (zoneId != null) { result.zoneId = zoneId } + [KEY_COLUMNS, KEY_FILTER, KEY_BINARY_AS_STRING, KEY_USE_COLUMN_INDEX_FILTER, KEY_LIMIT, KEY_ROW_GROUPS, KEY_ROW_GROUPS_BY_FILE, KEY_MAX_BATCH_ROWS, KEY_REMOTE_READ_MODE, KEY_CONNECT_TIMEOUT_MILLIS, KEY_READ_TIMEOUT_MILLIS, KEY_RANGE_CACHE_BYTES, KEY_SOURCE_FILE_COLUMN, KEY_PARALLELISM].each { String key -> + result[key] = this[key] + } result } static ParquetReadOptions fromMap(Map options) { ParquetReadOptions result = new ParquetReadOptions() Map normalized = OptionMaps.normalizeKeys(options) + Set known = descriptors()*.name*.toLowerCase(Locale.ROOT).toSet() + Set unknown = new HashSet<>(normalized.keySet()) + unknown.removeAll(known) + if (!unknown.empty) { throw new IllegalArgumentException("Unknown Parquet options: $unknown") } + normalized = normalized.findAll { String key, Object value -> value != null } if (normalized.containsKey('matrixname')) { String matrixName = OptionMaps.stringValueOrNull(normalized.matrixname) if (matrixName != null) { @@ -56,17 +156,51 @@ class ParquetReadOptions { result.zoneId(String.valueOf(value)) } } + applySelection(result, normalized) + result.validate() result } + private static void applySelection(ParquetReadOptions result, Map normalized) { + if (normalized.containsKey(KEY_COLUMNS)) { result.columns = (List) normalized.columns } + if (normalized.containsKey(KEY_FILTER)) { result.filter = (ParquetPredicate) normalized.filter } + if (normalized.containsKey('binaryasstring')) { result.binaryAsString = OptionMaps.booleanValueOrNull(normalized.binaryasstring, KEY_BINARY_AS_STRING) } + if (normalized.containsKey('usecolumnindexfilter')) { result.useColumnIndexFilter = OptionMaps.booleanValueOrNull(normalized.usecolumnindexfilter, KEY_USE_COLUMN_INDEX_FILTER) } + if (normalized.limit != null) { result.limit = ((Number) normalized.limit).longValue() } + if (normalized.rowgroupsbyfile != null) { result.rowGroupsByFile = (Map>) normalized.rowgroupsbyfile } + if (normalized.rowgroups != null) { result.rowGroups = (List) normalized.rowgroups } + if (normalized.maxbatchrows != null) { result.maxBatchRows = ((Number) normalized.maxbatchrows).intValue() } + if (normalized.remotereadmode != null) { result.remoteReadMode = String.valueOf(normalized.remotereadmode) } + if (normalized.connecttimeoutmillis != null) { result.connectTimeoutMillis = ((Number) normalized.connecttimeoutmillis).intValue() } + if (normalized.readtimeoutmillis != null) { result.readTimeoutMillis = ((Number) normalized.readtimeoutmillis).intValue() } + if (normalized.rangecachebytes != null) { result.rangeCacheBytes = ((Number) normalized.rangecachebytes).intValue() } + if (normalized.sourcefilecolumn != null) { result.sourceFileColumn = String.valueOf(normalized.sourcefilecolumn) } + if (normalized.parallelism != null) { result.parallelism = ((Number) normalized.parallelism).intValue() } + } + static String describe() { OptionDescriptor.describe(descriptors()) } static List descriptors() { + ParquetReadOptions defaults = new ParquetReadOptions() [ new OptionDescriptor('matrixName', String, null, 'Name for the resulting Matrix'), - new OptionDescriptor('zoneId', ZoneId, null, 'Time zone to use when reading timestamp values') + new OptionDescriptor('zoneId', ZoneId, null, 'Time zone to use when reading timestamp values'), + new OptionDescriptor(KEY_REMOTE_READ_MODE, String, defaults.remoteReadMode, 'Transport: DOWNLOAD, AUTO or RANGE'), + new OptionDescriptor(KEY_CONNECT_TIMEOUT_MILLIS, Integer, defaults.connectTimeoutMillis, 'HTTP connection timeout'), + new OptionDescriptor(KEY_READ_TIMEOUT_MILLIS, Integer, defaults.readTimeoutMillis, 'HTTP read timeout'), + new OptionDescriptor(KEY_RANGE_CACHE_BYTES, Integer, defaults.rangeCacheBytes, 'Maximum HTTP range cache bytes'), + new OptionDescriptor(KEY_SOURCE_FILE_COLUMN, String, null, 'Optional local source path column'), + new OptionDescriptor(KEY_PARALLELISM, Integer, defaults.parallelism, 'Bounded dataset decoder workers'), + new OptionDescriptor(KEY_COLUMNS, List, null, 'Ordered output columns'), + new OptionDescriptor(KEY_FILTER, ParquetPredicate, null, 'Exact logical row filter'), + new OptionDescriptor(KEY_BINARY_AS_STRING, Boolean, null, 'Unannotated BINARY override'), + new OptionDescriptor(KEY_USE_COLUMN_INDEX_FILTER, Boolean, defaults.useColumnIndexFilter, 'Enable page-index pruning'), + new OptionDescriptor(KEY_LIMIT, Long, null, 'Maximum accepted rows'), + new OptionDescriptor(KEY_ROW_GROUPS, List, null, 'Ordered original footer IDs'), + new OptionDescriptor(KEY_ROW_GROUPS_BY_FILE, Map, null, 'Per-file footer IDs keyed by canonical local path'), + new OptionDescriptor(KEY_MAX_BATCH_ROWS, Integer, null, 'Maximum rows per batch') ] } } diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetValueWriter.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetValueWriter.groovy index 5de9d92dc..6e093b1d1 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetValueWriter.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetValueWriter.groovy @@ -13,11 +13,13 @@ import org.apache.parquet.schema.Type import java.beans.Introspector import java.beans.PropertyDescriptor import java.math.RoundingMode +import java.nio.ByteBuffer import java.sql.Time import java.sql.Timestamp import java.time.Instant import java.time.LocalDate import java.time.LocalDateTime +import java.time.LocalTime import java.time.ZoneId import java.util.concurrent.ConcurrentHashMap @@ -120,7 +122,7 @@ final class ParquetValueWriter { static void writePrimitiveValue(Group group, String fieldName, PrimitiveType field, Object value, RoundingMode roundingMode) { switch (value.class) { - case Integer, int -> group.append(fieldName, (int) value) + case Byte, Short, Integer, int -> group.append(fieldName, (int) value) case Long, long -> group.append(fieldName, ((Number) value).longValue()) case BigInteger -> group.add(fieldName, Binary.fromConstantByteArray(((BigInteger) value).toByteArray())) case Float, float -> group.append(fieldName, ((Number) value).floatValue()) @@ -132,8 +134,23 @@ final class ParquetValueWriter { case Time -> group.append(fieldName, timeToMillis((Time) value)) case LocalDateTime -> group.append(fieldName, instantToMicros(((LocalDateTime) value).atZone(getZoneId()).toInstant())) case Timestamp -> group.append(fieldName, instantToMicros(((Timestamp) value).toInstant())) + case byte[] -> group.add(fieldName, Binary.fromConstantByteArray((byte[]) value)) + case LocalTime -> group.append(fieldName, ((LocalTime) value).toNanoOfDay()) + case Instant -> { + Instant instant = (Instant) value + if (instant.nano % NANOS_PER_MICRO != 0) { + throw new IllegalArgumentException("Instant field '$fieldName' loses sub-microsecond precision") + } + group.append(fieldName, instantToMicros(instant)) + } + case UUID -> { + UUID uuid = (UUID) value + group.add(fieldName, Binary.fromConstantByteArray(ByteBuffer.allocate(16).putLong(uuid.mostSignificantBits).putLong(uuid.leastSignificantBits).array())) + } + case String, Character, CharSequence -> group.append(fieldName, value.toString()) + case Enum -> group.append(fieldName, ((Enum) value).name()) case Date -> group.append(fieldName, ((Date) value).time) - default -> group.append(fieldName, value.toString()) + default -> throw new IllegalArgumentException("Unsupported ${value.getClass().name} for field $fieldName") } } @@ -143,7 +160,7 @@ final class ParquetValueWriter { } private static long instantToMicros(Instant instant) { - instant.epochSecond * MICROS_PER_SECOND + (long) (instant.nano / NANOS_PER_MICRO) + Math.addExact(Math.multiplyExact(instant.epochSecond, MICROS_PER_SECOND), (long) (instant.nano / NANOS_PER_MICRO)) } private static void writeBigDecimalValue(Group group, String fieldName, PrimitiveType field, BigDecimal value, @@ -154,7 +171,7 @@ final class ParquetValueWriter { writeFixedLenBigDecimal(group, fieldName, field, value, (LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) logical, roundingMode) } else { - group.append(fieldName, value.doubleValue()) + throw new IllegalArgumentException("Missing DECIMAL annotation for $fieldName") } } diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetWriteOptions.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetWriteOptions.groovy index 84a085dda..8aec792f2 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetWriteOptions.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetWriteOptions.groovy @@ -12,6 +12,12 @@ import java.time.ZoneId * Typed options for Parquet write operations via the SPI. */ class ParquetWriteOptions { + private static final String KEY_DICTIONARY_ENCODING = 'dictionaryEncoding' + private static final String KEY_WRITER_VERSION = 'writerVersion' + private static final String KEY_SIZE_STATISTICS_ENABLED = 'sizeStatisticsEnabled' + + private static final String KEY_V1 = 'V1' + private static final String KEY_PRECISION = 'precision' private static final String KEY_SCALE = 'scale' @@ -19,6 +25,51 @@ class ParquetWriteOptions { private static final String KEY_ROUNDING_MODE = 'roundingMode' private static final String ERR_ROUNDING_MODE_NULL = 'roundingMode cannot be null' + long rowGroupSize = 134217728L + + /** Sets rowGroupSize (default 134217728L); target sizes are not hard memory limits. */ + ParquetWriteOptions rowGroupSize(long value) { this.rowGroupSize = value; this } + + int pageSize = 1048576 + + /** Sets pageSize (default 1048576); target sizes are not hard memory limits. */ + ParquetWriteOptions pageSize(int value) { this.pageSize = value; this } + + boolean dictionaryEncoding = true + + /** Enable dictionary encoding with plain-page fallback (default true). */ + ParquetWriteOptions dictionaryEncoding(boolean value) { this.dictionaryEncoding = value; this } + + String writerVersion = KEY_V1 + + /** Choose V1 or V2 value encodings and page layout (default V1). */ + ParquetWriteOptions writerVersion(String value) { this.writerVersion = value; this } + + int columnIndexTruncateLength = 64 + + /** Maximum binary bound bytes in page indexes (default 64). */ + ParquetWriteOptions columnIndexTruncateLength(int value) { this.columnIndexTruncateLength = value; this } + + int statisticsTruncateLength = Integer.MAX_VALUE + + /** Maximum binary bound bytes in row-group statistics (default 2147483647). */ + ParquetWriteOptions statisticsTruncateLength(int value) { this.statisticsTruncateLength = value; this } + + Map bloomFilterEnabled = [:] + + /** Per-primitive-column bloom overrides (default empty, disabled). */ + ParquetWriteOptions bloomFilterEnabled(Map value) { this.bloomFilterEnabled = value; this } + + Map bloomFilterNDV = [:] + + /** Positive expected distinct counts for bloom sizing (default empty). */ + ParquetWriteOptions bloomFilterNDV(Map value) { this.bloomFilterNDV = value; this } + + boolean sizeStatisticsEnabled = true + + /** Collect column size statistics (default true). */ + ParquetWriteOptions sizeStatisticsEnabled(boolean value) { this.sizeStatisticsEnabled = value; this } + boolean inferPrecisionAndScale = true Integer precision = null Integer scale = null @@ -120,6 +171,14 @@ class ParquetWriteOptions { } void validate() { + if (rowGroupSize <= 0 || pageSize <= 0 || columnIndexTruncateLength <= 0 || statisticsTruncateLength <= 0) { + throw new IllegalArgumentException('Writer sizes/truncate lengths must be positive') + } + if (!(writerVersion in [KEY_V1, 'V2'])) { throw new IllegalArgumentException('writerVersion must be V1 or V2') } + if (bloomFilterEnabled == null || bloomFilterNDV == null) { throw new IllegalArgumentException('Bloom option maps cannot be null') } + if (bloomFilterEnabled.values().any { it == null }) { throw new IllegalArgumentException('Bloom overrides must be non-null Booleans') } + if (bloomFilterNDV.values().any { it == null || it <= 0 }) { throw new IllegalArgumentException('Bloom NDVs must be positive') } + if ((precision == null) != (scale == null)) { throw new IllegalArgumentException('precision and scale must be provided together') } @@ -157,12 +216,19 @@ class ParquetWriteOptions { if (zoneId != null) { result.zoneId = zoneId } + result.putAll([rowGroupSize: rowGroupSize, pageSize: pageSize, dictionaryEncoding: dictionaryEncoding, writerVersion: writerVersion, columnIndexTruncateLength: columnIndexTruncateLength, statisticsTruncateLength: statisticsTruncateLength, bloomFilterEnabled: bloomFilterEnabled, bloomFilterNDV: bloomFilterNDV, sizeStatisticsEnabled: sizeStatisticsEnabled]) result } + @SuppressWarnings('AbcMetric') // Explicit typed option normalization. static ParquetWriteOptions fromMap(Map options) { ParquetWriteOptions result = new ParquetWriteOptions() Map normalized = OptionMaps.normalizeKeys(options) + Set known = descriptors()*.name*.toLowerCase(Locale.ROOT).toSet() + Set unknown = new HashSet<>(normalized.keySet()) + unknown.removeAll(known) + if (!unknown.empty) { throw new IllegalArgumentException("Unknown Parquet options: $unknown") } + normalized = normalized.findAll { String key, Object value -> value != null } if (normalized.containsKey('inferprecisionandscale')) { Object inferPrecisionAndScale = normalized.inferprecisionandscale if (inferPrecisionAndScale != null) { @@ -216,6 +282,15 @@ class ParquetWriteOptions { result.roundingMode(String.valueOf(value)) } } + if (normalized.containsKey('rowgroupsize')) { result.rowGroupSize = ((Number) normalized.rowgroupsize).longValue() } + if (normalized.containsKey('pagesize')) { result.pageSize = ((Number) normalized.pagesize).intValue() } + if (normalized.containsKey('dictionaryencoding')) { result.dictionaryEncoding = OptionMaps.booleanValueOrNull(normalized.dictionaryencoding, KEY_DICTIONARY_ENCODING) } + if (normalized.containsKey('writerversion')) { result.writerVersion = String.valueOf(normalized.writerversion) } + if (normalized.containsKey('columnindextruncatelength')) { result.columnIndexTruncateLength = ((Number) normalized.columnindextruncatelength).intValue() } + if (normalized.containsKey('statisticstruncatelength')) { result.statisticsTruncateLength = ((Number) normalized.statisticstruncatelength).intValue() } + if (normalized.containsKey('bloomfilterenabled')) { result.bloomFilterEnabled = (Map) normalized.bloomfilterenabled } + if (normalized.containsKey('bloomfilterndv')) { result.bloomFilterNDV = (Map) normalized.bloomfilterndv } + if (normalized.containsKey('sizestatisticsenabled')) { result.sizeStatisticsEnabled = OptionMaps.booleanValueOrNull(normalized.sizestatisticsenabled, KEY_SIZE_STATISTICS_ENABLED) } result.validate() result } @@ -281,6 +356,7 @@ class ParquetWriteOptions { } static List descriptors() { + ParquetWriteOptions defaults = new ParquetWriteOptions() [ new OptionDescriptor(KEY_INFER_PRECISION_AND_SCALE, Boolean, Boolean.TRUE.toString(), 'Infer precision and scale for BigDecimal columns'), new OptionDescriptor(KEY_PRECISION, Integer, null, 'Uniform precision for all BigDecimal columns'), @@ -288,7 +364,16 @@ class ParquetWriteOptions { new OptionDescriptor('decimalMeta', Map, null, 'Map of column names to [precision, scale] arrays'), new OptionDescriptor('zoneId', ZoneId, null, 'Time zone to use when writing timestamp values'), new OptionDescriptor('compressionCodec', CompressionCodecName, CompressionCodecName.SNAPPY.name(), 'Compression codec for the Parquet file (e.g. SNAPPY, GZIP, ZSTD, UNCOMPRESSED)'), - new OptionDescriptor(KEY_ROUNDING_MODE, RoundingMode, RoundingMode.UNNECESSARY.name(), 'Rounding mode for explicit DECIMAL scale conversion') + new OptionDescriptor(KEY_ROUNDING_MODE, RoundingMode, RoundingMode.UNNECESSARY.name(), 'Rounding mode for explicit DECIMAL scale conversion'), + new OptionDescriptor('rowGroupSize', Long, defaults.rowGroupSize, 'Target row-group bytes'), + new OptionDescriptor('pageSize', Integer, defaults.pageSize, 'Target data-page bytes'), + new OptionDescriptor(KEY_DICTIONARY_ENCODING, Boolean, defaults.dictionaryEncoding, 'Enable dictionary encoding'), + new OptionDescriptor(KEY_WRITER_VERSION, String, defaults.writerVersion, 'V1 or V2 encodings and page layout'), + new OptionDescriptor('columnIndexTruncateLength', Integer, defaults.columnIndexTruncateLength, 'Truncated binary page bounds bytes'), + new OptionDescriptor('statisticsTruncateLength', Integer, defaults.statisticsTruncateLength, 'Truncated row-group binary statistics bytes'), + new OptionDescriptor('bloomFilterEnabled', Map, null, 'Per-column bloom-filter enabled overrides'), + new OptionDescriptor('bloomFilterNDV', Map, null, 'Per-column positive expected distinct values'), + new OptionDescriptor(KEY_SIZE_STATISTICS_ENABLED, Boolean, defaults.sizeStatisticsEnabled, 'Collect column size statistics') ] } } diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/Predicates.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/Predicates.groovy new file mode 100644 index 000000000..f4432573f --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/Predicates.groovy @@ -0,0 +1,276 @@ +package se.alipsa.matrix.parquet + +import java.time.* + +/** Factory for immutable, schema-aware predicates. Null comparisons use explicit null tests. */ +final class Predicates { + private static final String KEY_EQ = 'eq' + private static final String KEY_NE = 'ne' + private static final String KEY_LT = 'lt' + private static final String KEY_LE = 'le' + private static final String KEY_GT = 'gt' + private static final String KEY_GE = 'ge' + + private Predicates() {} + + /** Compares a column using eq; values must match its logical schema. */ + static ParquetPredicate eq(String column, Number value) { + new ParquetPredicate(KEY_EQ, column, value) + } + + /** Compares a column using eq; values must match its logical schema. */ + static ParquetPredicate eq(String column, String value) { + new ParquetPredicate(KEY_EQ, column, value) + } + + /** Compares a column using eq; values must match its logical schema. */ + static ParquetPredicate eq(String column, Boolean value) { + new ParquetPredicate(KEY_EQ, column, value) + } + + /** Compares a column using eq; values must match its logical schema. */ + static ParquetPredicate eq(String column, LocalDate value) { + new ParquetPredicate(KEY_EQ, column, value) + } + + /** Compares a column using eq; values must match its logical schema. */ + static ParquetPredicate eq(String column, LocalDateTime value) { + new ParquetPredicate(KEY_EQ, column, value) + } + + /** Compares a column using eq; values must match its logical schema. */ + static ParquetPredicate eq(String column, Instant value) { + new ParquetPredicate(KEY_EQ, column, value) + } + + /** Compares a column using eq; values must match its logical schema. */ + static ParquetPredicate eq(String column, LocalTime value) { + new ParquetPredicate(KEY_EQ, column, value) + } + + /** Compares a column using eq; values must match its logical schema. */ + static ParquetPredicate eq(String column, byte[] value) { + new ParquetPredicate(KEY_EQ, column, value) + } + + /** Compares a column using ne; values must match its logical schema. */ + static ParquetPredicate ne(String column, Number value) { + new ParquetPredicate(KEY_NE, column, value) + } + + /** Compares a column using ne; values must match its logical schema. */ + static ParquetPredicate ne(String column, String value) { + new ParquetPredicate(KEY_NE, column, value) + } + + /** Compares a column using ne; values must match its logical schema. */ + static ParquetPredicate ne(String column, Boolean value) { + new ParquetPredicate(KEY_NE, column, value) + } + + /** Compares a column using ne; values must match its logical schema. */ + static ParquetPredicate ne(String column, LocalDate value) { + new ParquetPredicate(KEY_NE, column, value) + } + + /** Compares a column using ne; values must match its logical schema. */ + static ParquetPredicate ne(String column, LocalDateTime value) { + new ParquetPredicate(KEY_NE, column, value) + } + + /** Compares a column using ne; values must match its logical schema. */ + static ParquetPredicate ne(String column, Instant value) { + new ParquetPredicate(KEY_NE, column, value) + } + + /** Compares a column using ne; values must match its logical schema. */ + static ParquetPredicate ne(String column, LocalTime value) { + new ParquetPredicate(KEY_NE, column, value) + } + + /** Compares a column using ne; values must match its logical schema. */ + static ParquetPredicate ne(String column, byte[] value) { + new ParquetPredicate(KEY_NE, column, value) + } + + /** Compares a column using lt; values must match its logical schema. */ + static ParquetPredicate lt(String column, Number value) { + new ParquetPredicate(KEY_LT, column, value) + } + + /** Compares a column using lt; values must match its logical schema. */ + static ParquetPredicate lt(String column, String value) { + new ParquetPredicate(KEY_LT, column, value) + } + + /** Compares a column using lt; values must match its logical schema. */ + static ParquetPredicate lt(String column, Boolean value) { + new ParquetPredicate(KEY_LT, column, value) + } + + /** Compares a column using lt; values must match its logical schema. */ + static ParquetPredicate lt(String column, LocalDate value) { + new ParquetPredicate(KEY_LT, column, value) + } + + /** Compares a column using lt; values must match its logical schema. */ + static ParquetPredicate lt(String column, LocalDateTime value) { + new ParquetPredicate(KEY_LT, column, value) + } + + /** Compares a column using lt; values must match its logical schema. */ + static ParquetPredicate lt(String column, Instant value) { + new ParquetPredicate(KEY_LT, column, value) + } + + /** Compares a column using lt; values must match its logical schema. */ + static ParquetPredicate lt(String column, LocalTime value) { + new ParquetPredicate(KEY_LT, column, value) + } + + /** Compares a column using lt; values must match its logical schema. */ + static ParquetPredicate lt(String column, byte[] value) { + new ParquetPredicate(KEY_LT, column, value) + } + + /** Compares a column using le; values must match its logical schema. */ + static ParquetPredicate le(String column, Number value) { + new ParquetPredicate(KEY_LE, column, value) + } + + /** Compares a column using le; values must match its logical schema. */ + static ParquetPredicate le(String column, String value) { + new ParquetPredicate(KEY_LE, column, value) + } + + /** Compares a column using le; values must match its logical schema. */ + static ParquetPredicate le(String column, Boolean value) { + new ParquetPredicate(KEY_LE, column, value) + } + + /** Compares a column using le; values must match its logical schema. */ + static ParquetPredicate le(String column, LocalDate value) { + new ParquetPredicate(KEY_LE, column, value) + } + + /** Compares a column using le; values must match its logical schema. */ + static ParquetPredicate le(String column, LocalDateTime value) { + new ParquetPredicate(KEY_LE, column, value) + } + + /** Compares a column using le; values must match its logical schema. */ + static ParquetPredicate le(String column, Instant value) { + new ParquetPredicate(KEY_LE, column, value) + } + + /** Compares a column using le; values must match its logical schema. */ + static ParquetPredicate le(String column, LocalTime value) { + new ParquetPredicate(KEY_LE, column, value) + } + + /** Compares a column using le; values must match its logical schema. */ + static ParquetPredicate le(String column, byte[] value) { + new ParquetPredicate(KEY_LE, column, value) + } + + /** Compares a column using gt; values must match its logical schema. */ + static ParquetPredicate gt(String column, Number value) { + new ParquetPredicate(KEY_GT, column, value) + } + + /** Compares a column using gt; values must match its logical schema. */ + static ParquetPredicate gt(String column, String value) { + new ParquetPredicate(KEY_GT, column, value) + } + + /** Compares a column using gt; values must match its logical schema. */ + static ParquetPredicate gt(String column, Boolean value) { + new ParquetPredicate(KEY_GT, column, value) + } + + /** Compares a column using gt; values must match its logical schema. */ + static ParquetPredicate gt(String column, LocalDate value) { + new ParquetPredicate(KEY_GT, column, value) + } + + /** Compares a column using gt; values must match its logical schema. */ + static ParquetPredicate gt(String column, LocalDateTime value) { + new ParquetPredicate(KEY_GT, column, value) + } + + /** Compares a column using gt; values must match its logical schema. */ + static ParquetPredicate gt(String column, Instant value) { + new ParquetPredicate(KEY_GT, column, value) + } + + /** Compares a column using gt; values must match its logical schema. */ + static ParquetPredicate gt(String column, LocalTime value) { + new ParquetPredicate(KEY_GT, column, value) + } + + /** Compares a column using gt; values must match its logical schema. */ + static ParquetPredicate gt(String column, byte[] value) { + new ParquetPredicate(KEY_GT, column, value) + } + + /** Compares a column using ge; values must match its logical schema. */ + static ParquetPredicate ge(String column, Number value) { + new ParquetPredicate(KEY_GE, column, value) + } + + /** Compares a column using ge; values must match its logical schema. */ + static ParquetPredicate ge(String column, String value) { + new ParquetPredicate(KEY_GE, column, value) + } + + /** Compares a column using ge; values must match its logical schema. */ + static ParquetPredicate ge(String column, Boolean value) { + new ParquetPredicate(KEY_GE, column, value) + } + + /** Compares a column using ge; values must match its logical schema. */ + static ParquetPredicate ge(String column, LocalDate value) { + new ParquetPredicate(KEY_GE, column, value) + } + + /** Compares a column using ge; values must match its logical schema. */ + static ParquetPredicate ge(String column, LocalDateTime value) { + new ParquetPredicate(KEY_GE, column, value) + } + + /** Compares a column using ge; values must match its logical schema. */ + static ParquetPredicate ge(String column, Instant value) { + new ParquetPredicate(KEY_GE, column, value) + } + + /** Compares a column using ge; values must match its logical schema. */ + static ParquetPredicate ge(String column, LocalTime value) { + new ParquetPredicate(KEY_GE, column, value) + } + + /** Compares a column using ge; values must match its logical schema. */ + static ParquetPredicate ge(String column, byte[] value) { + new ParquetPredicate(KEY_GE, column, value) + } + + /** Tests whether the column is null. */ + static ParquetPredicate isNull(String column) { new ParquetPredicate('isNull', column, null) } + + /** Tests whether the column is non-null. */ + static ParquetPredicate isNotNull(String column) { new ParquetPredicate('isNotNull', column, null) } + + /** Combines two predicates with AND. */ + static ParquetPredicate and(ParquetPredicate left, ParquetPredicate right) { + new ParquetPredicate('and', null, null, [left, right]) + } + + /** Combines two predicates with OR. */ + static ParquetPredicate or(ParquetPredicate left, ParquetPredicate right) { + new ParquetPredicate('or', null, null, [left, right]) + } + + /** Negates an exact predicate, including its documented null behavior. */ + static ParquetPredicate not(ParquetPredicate predicate) { + new ParquetPredicate('not', null, null, [predicate]) + } +} diff --git a/matrix-parquet/src/test/groovy/HubUrlTest.groovy b/matrix-parquet/src/test/groovy/HubUrlTest.groovy new file mode 100644 index 000000000..4f67ff431 --- /dev/null +++ b/matrix-parquet/src/test/groovy/HubUrlTest.groovy @@ -0,0 +1,30 @@ +import static org.junit.jupiter.api.Assertions.* + +import org.junit.jupiter.api.Test + +import se.alipsa.matrix.parquet.HubUrl + +/** Dataset cases pinned to huggingface_hub v2.2.0 parse_hf_uri and file URL quoting. */ +class HubUrlTest { + @Test + void revisionsAndLiteralFileNames() { + new File('src/test/resources/optimized/hub-urls.tsv').readLines('UTF-8') + .findAll { !it.startsWith('#') }.each { String line -> + String[] parts = line.split('\t', 2) + assertEquals(parts[1], HubUrl.resolve(parts[0]).toString()) + } + ['hf://models/owner/repo/file', 'hf://datasets/owner/repo@/file', 'hf://datasets/owner/repo'].each { value -> + assertThrows(IllegalArgumentException) { HubUrl.resolve(value) } + } + } + + @Test + void rejectDotSegments() { + ['.', '..', 'nested/../file', 'nested/./file'].each { path -> + assertThrows(IllegalArgumentException) { HubUrl.resolve("hf://datasets/owner/repo/$path") } + } + ['..', '%2E%2E', 'feature%2F..%2Fbranch'].each { revision -> + assertThrows(IllegalArgumentException) { HubUrl.resolve("hf://datasets/owner/repo@$revision/file") } + } + } +} diff --git a/matrix-parquet/src/test/groovy/MatrixParquetTest.groovy b/matrix-parquet/src/test/groovy/MatrixParquetTest.groovy index c7caa3d19..d270cb4b7 100644 --- a/matrix-parquet/src/test/groovy/MatrixParquetTest.groovy +++ b/matrix-parquet/src/test/groovy/MatrixParquetTest.groovy @@ -114,7 +114,7 @@ class MatrixParquetTest { def empData = Matrix.builder('empData').columns( emp_id: 1..5, emp_name: ['Rick', 'Dan', 'Michelle', 'Ryan', 'Gary'], - salary: [623.3, 515.2, 611, 729.0, 843.25], + salary: [623.3, 515.2, 611.0, 729.0, 843.25], score: toBigDecimals(['1.2345', 0.6321, 0.7190, 0.8452, 0.9198]), start_date: toLocalDates('2012-01-01', '2013-09-23', '2014-11-15', '2014-05-11', '2015-03-27')) .types([int, String, BigDecimal, BigDecimal, LocalDate]).build() @@ -444,7 +444,7 @@ class MatrixParquetTest { def dirEx = assertThrows(IllegalArgumentException) { MatrixParquetReader.read(dir) } - assertTrue(dirEx.message.contains('directory')) + assertTrue(dirEx.message.contains('nonempty')) } @Test @@ -739,8 +739,8 @@ class MatrixParquetTest { value: toBigDecimals([12.34, 56.78]) ).types([BigDecimal]).build() - byte[] parquetBytes = MatrixParquetWriter.writeBytes(data, false) - assert parquetBytes != null + assertThrows(IllegalArgumentException) { MatrixParquetWriter.writeBytes(data, false) } + byte[] parquetBytes = MatrixParquetWriter.writeBytes(data, [value: [4, 2] as int[]]) assert parquetBytes.length > 0 } @@ -824,7 +824,7 @@ class MatrixParquetTest { void testWriterBuilderWithPrecisionAndScale() { def data = Matrix.builder('empData').columns( emp_id: 1..5, - salary: [623.3, 515.2, 611, 729.0, 843.25], + salary: [623.3, 515.2, 611.0, 729.0, 843.25], score: toBigDecimals(['1.2345', 0.6321, 0.7190, 0.8452, 0.9198]) ).types([int, BigDecimal, BigDecimal]).build() @@ -899,10 +899,11 @@ class MatrixParquetTest { value: toBigDecimals([12.34, 56.78]) ).types([BigDecimal]).build() - byte[] bytes = MatrixParquetWriter.builder(data) - .inferPrecisionAndScale(false) - .writeBytes() - assert bytes != null + assertThrows(IllegalArgumentException) { + MatrixParquetWriter.builder(data).inferPrecisionAndScale(false).writeBytes() + } + byte[] bytes = MatrixParquetWriter.builder(data).inferPrecisionAndScale(false) + .precision(8).scale(2).writeBytes() assert bytes.length > 0 } diff --git a/matrix-parquet/src/test/groovy/OptimizedParquetTest.groovy b/matrix-parquet/src/test/groovy/OptimizedParquetTest.groovy new file mode 100644 index 000000000..90518d886 --- /dev/null +++ b/matrix-parquet/src/test/groovy/OptimizedParquetTest.groovy @@ -0,0 +1,129 @@ +import static org.junit.jupiter.api.Assertions.* + +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import se.alipsa.matrix.core.Matrix +import se.alipsa.matrix.parquet.* + +import java.time.* + +class OptimizedParquetTest { + @TempDir File directory + + @Test + void externalCdcAndIndexes() { + [0, 1].each { cdc -> + [0, 1].each { index -> + File file = new File('src/test/resources/optimized', "cdc-${cdc}-index-${index}.parquet") + def result = MatrixParquetReader.builder().zoneId('Europe/Stockholm').read(file) + assertEquals(4096, result.rowCount()) + assertEquals(LocalDateTime.of(2024, 3, 31, 1, 30), result[0, 'local']) + assertArrayEquals([0, 0, -1] as byte[], (byte[]) result[0, 'payload']) + assertEquals(1.23G, result[123, 'price']) + assertEquals(['α', null, 'β'], result[1, 'tags']) + assertArrayEquals([1, 0, -1] as byte[], (byte[]) result[1, 'media'].bytes) + assertEquals('image-1.jpg', result[1, 'media'].path) + assertNull(result[2048, 'nulls']) + assertEquals('café-3-' + 'abcd' * 32, result[4095, 'text']) + [true, false].each { useIndex -> + def selected = MatrixParquetReader.builder().columns(['text', 'id']) + .filter(Predicates.ge('id', 3000L)).limit(13).useColumnIndexFilter(useIndex).read(file) + assertEquals(['text', 'id'], selected.columnNames()) + assertEquals((3000L..<3013L).toList(), selected.column('id')) + } + def groups = MatrixParquetReader.builder().rowGroups([1, 0]).limit(3).read(file) + assertEquals([2048L, 2049L, 2050L], groups.column('id')) + MatrixParquetReader.builder().maxBatchRows(127).batches(file).withCloseable { batches -> + int count = 0 + while (batches.hasNext()) { + Matrix batch = batches.next() + assertTrue(batch.rowCount() <= 127) + count += batch.rowCount() + } + assertEquals(4096, count) + } + } + } + } + + @Test + void alternatePairwiseFixtures() { + new File('src/test/resources/optimized').listFiles().findAll { it.name.endsWith('-alternate.parquet') }.each { file -> + def data = MatrixParquetReader.read(file) + assertEquals(4096, data.rowCount()) + assertEquals(1.23G, data[123, 'price']) + assertEquals(['α', null, 'β'], data[1, 'tags']) + } + } + + @Test + @SuppressWarnings('ClassForName') // Characterize JVM array-name lookup; production uses a fixed allowlist. + void fixtureEntryPointsAndLegacyCharacterization() { + File file = new File('src/test/resources/optimized/cdc-1-index-1.parquet') + [MatrixParquetReader.read(file.bytes), MatrixParquetReader.read(new ByteArrayInputStream(file.bytes)), + MatrixParquetReader.read(file.toURI().toURL()), new ParquetFormatProvider().read(file, [:])].each { result -> + assertEquals(4096, result.rowCount()) + assertEquals(['α', null, 'β'], result[1, 'tags']) + assertArrayEquals([0, 0, -1] as byte[], (byte[]) result[0, 'payload']) + } + def legacy = MatrixParquetReader.read(new File('src/test/resources/optimized/matrix-0.7-date-units.parquet')) + assertEquals(1000L, legacy[0, 'value'].time) + assertEquals(2_000_000L, legacy[1, 'value'].time) // Original Timestamp(2000) was written as micros into MILLIS. + def metadata = MatrixParquetReader.builder().zoneId('UTC').read(new File('src/test/resources/optimized/matrix-0.7-metadata.parquet')) + assertEquals(BigDecimal, metadata.type('amount')) + assertEquals(LocalDateTime.of(2024, 1, 1, 0, 0), metadata[0, 'local']) + assertThrows(IllegalArgumentException) { + MatrixParquetReader.read(new File('src/test/resources/optimized/legacy-binary-metadata.parquet')) + } + assertThrows(ClassNotFoundException) { getClass().classLoader.loadClass('[B') } + assertEquals(byte[], Class.forName('[B', false, getClass().classLoader)) + } + + @Test + void canonicalTypesAndBinaryRoundTrip() { + def uuid = UUID.fromString('00112233-4455-6677-8899-aabbccddeeff') + def instant = Instant.parse('1500-01-01T00:00:00.123456Z') + def data = Matrix.builder('policies').columns( + id: [1, 2], text: ['α', null], price: [1.5G, 2.25G], + binary: [[0, -1] as byte[], [] as byte[]], tiny: [(byte) 1, (byte) -2], + small: [(short) 1000, (short) -1000], time: [LocalTime.NOON, null], + instant: [instant, null], uuid: [uuid, null], nested: [[label: 'hello'], [:]]) + .build() + byte[] bytes = MatrixParquetWriter.writeBytes(data) + def result = MatrixParquetReader.read(bytes) + assertEquals(BigDecimal, result.type('price')) + assertEquals([1.50G, 2.25G], result.column('price')) + assertArrayEquals([0, -1] as byte[], (byte[]) result[0, 'binary']) + assertEquals(Byte, result.type('tiny')) + assertEquals((byte) -2, result[1, 'tiny']) + assertEquals(LocalTime.NOON, result[0, 'time']) + assertEquals(instant, result[0, 'instant']) + assertEquals(uuid, result[0, 'uuid']) + assertEquals([label: 'hello'], result[0, 'nested']) + } + + @Test + void validationBeforeWriting() { + File target = new File(directory, 'untouched.parquet') + target.text = 'keep' + [Matrix.builder().columns(value: [1, 'x']).build(), + Matrix.builder().columns(value: [new Date(0), new java.sql.Timestamp(0)]).types(Date).build(), + Matrix.builder().columns(value: [1]).types(BigDecimal).build()].each { data -> + assertThrows(IllegalArgumentException) { MatrixParquetWriter.write(data, target) } + assertEquals('keep', target.text) + } + } + + @Test + void legacyStringsAndWriterTuning() { + def old = MatrixParquetReader.read(new File('src/test/resources/optimized/matrix-0.7-strings.parquet')) + assertEquals('héllo', old[0, 'text']) + assertEquals([label: 'α'], old[0, 'nested']) + def data = Matrix.builder().columns(id: (0..<10000).toList(), text: ['label'] * 10000).build() + File file = new File(directory, 'tuned.parquet') + MatrixParquetWriter.builder(data).pageSize(4096).rowGroupSize(32768).writerVersion('V2') + .bloomFilterEnabled([id: true]).bloomFilterNDV([id: 10000L]).write(file) + assertEquals(10000, MatrixParquetReader.read(file).rowCount()) + } +} diff --git a/matrix-parquet/src/test/groovy/ParquetBaselineTest.groovy b/matrix-parquet/src/test/groovy/ParquetBaselineTest.groovy new file mode 100644 index 000000000..67fd0e25a --- /dev/null +++ b/matrix-parquet/src/test/groovy/ParquetBaselineTest.groovy @@ -0,0 +1,27 @@ +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfSystemProperty + +import se.alipsa.matrix.core.Matrix +import se.alipsa.matrix.parquet.MatrixParquetReader +import se.alipsa.matrix.parquet.MatrixParquetWriter + +/** Opt-in baseline and legacy fixture capture, run before changing the writer. */ +@EnabledIfSystemProperty(named = 'parquet.baseline', matches = 'true') +class ParquetBaselineTest { + @Test + void capture() { + int rows = 1_000_000 + def columns = [id: (0.. columns["text$c".toString()] = (0.. + long start = System.nanoTime() + def result = MatrixParquetReader.read(file) + long millis = (System.nanoTime() - start).intdiv(1_000_000) + assert result.rowCount() == rows + println "baseline wide run=$run rows=$rows fileBytes=${file.length()} millis=$millis" + } + } +} diff --git a/matrix-parquet/src/test/groovy/ParquetCodecProbeTest.groovy b/matrix-parquet/src/test/groovy/ParquetCodecProbeTest.groovy new file mode 100644 index 000000000..3a4d7ef7b --- /dev/null +++ b/matrix-parquet/src/test/groovy/ParquetCodecProbeTest.groovy @@ -0,0 +1,25 @@ +import static org.junit.jupiter.api.Assertions.assertEquals + +import org.junit.jupiter.api.Test + +import se.alipsa.matrix.core.Matrix +import se.alipsa.matrix.parquet.MatrixParquetReader +import se.alipsa.matrix.parquet.MatrixParquetWriter + +/** Normal codec round trips; the opt-in no-Hadoop lane also reports required runtime classes. */ +class ParquetCodecProbeTest { + @Test + void codecs() { + ['SNAPPY', 'ZSTD', 'GZIP', 'UNCOMPRESSED'].each { codec -> + try { + def matrix = Matrix.builder('codec').columns(id: [1, 2], text: ['a', 'b']).build() + byte[] bytes = MatrixParquetWriter.builder(matrix).compressionCodec(codec).writeBytes() + assertEquals([1, 2], MatrixParquetReader.read(bytes).column('id')) + println "CODEC $codec OK" + } catch (NoClassDefFoundError failure) { + if (System.getProperty('parquet.codecSpike') != 'true') { throw failure } + println "CODEC $codec requires ${failure.message}" + } + } + } +} diff --git a/matrix-parquet/src/test/groovy/ParquetDatasetAndHttpTest.groovy b/matrix-parquet/src/test/groovy/ParquetDatasetAndHttpTest.groovy new file mode 100644 index 000000000..257de97a9 --- /dev/null +++ b/matrix-parquet/src/test/groovy/ParquetDatasetAndHttpTest.groovy @@ -0,0 +1,174 @@ +import static org.junit.jupiter.api.Assertions.* + +import com.sun.net.httpserver.HttpServer +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import se.alipsa.matrix.core.Matrix +import se.alipsa.matrix.parquet.* + +import java.util.concurrent.atomic.AtomicLong + +class ParquetDatasetAndHttpTest { + @TempDir File directory + + @Test + void orderedShardsParallelLimitsAndSchemaChecks() { + File first = new File(directory, 'a.parquet') + File second = new File(directory, 'b.parquet') + [first, second].eachWithIndex { file, index -> + MatrixParquetWriter.write(Matrix.builder().columns(id: [index * 2, index * 2 + 1]).build(), file) + } + [1, 2].each { workers -> + def data = MatrixParquetReader.builder().parallelism(workers).sourceFileColumn('source') + .limit(3).read([second, first]) + assertEquals([2, 3, 0], data.column('id')) + assertEquals([second.canonicalPath, second.canonicalPath, first.canonicalPath], data.column('source')) + } + def early = MatrixParquetReader.builder().parallelism(2).maxBatchRows(1).batches([first, second]) + assertEquals([0], early.next().column('id')) + early.close() + assertFalse(early.hasNext()) + early.close() + assertEquals([0, 1, 2, 3], MatrixParquetReader.read(directory).column('id')) + assertEquals([0, 1, 2, 3], MatrixParquetReader.readGlob("${directory.absolutePath}/*.parquet").column('id')) + assertEquals(['id'], MatrixParquetReader.builder().limit(0).read([first, second]).columnNames()) + assertEquals(0, MatrixParquetReader.builder().filter(Predicates.lt('id', 0)).read([first, second]).rowCount()) + def perFile = MatrixParquetReader.builder().parallelism(2) + .rowGroupsByFile([(first.canonicalPath): [], (second.canonicalPath): [0]]).read([first, second]) + assertEquals([2, 3], perFile.column('id')) + assertThrows(IllegalArgumentException) { MatrixParquetReader.read([first, first]) } + MatrixParquetWriter.write(Matrix.builder().columns(id: ['incompatible']).build(), second) + assertThrows(IllegalArgumentException) { MatrixParquetReader.read([first, second]) } + } + + @Test + void parallelRowGroupsKeepOrderAndMatchingLimit() { + File file = new File('src/test/resources/optimized/cdc-1-index-1.parquet') + [1, 2].each { workers -> + def result = MatrixParquetReader.builder().parallelism(workers).columns(['id']) + .filter(Predicates.ge('id', 2040L)).maxBatchRows(7).limit(20).read([file]) + assertEquals((2040L..<2060L).toList(), result.column('id')) + } + } + + @Test + void bulkRangesUseOneRequestAndReuseRedirectAndCache() { + byte[] bytes = new byte[2 * 1024 * 1024] + Arrays.fill(bytes, (byte) 37) + AtomicLong redirects = new AtomicLong() + AtomicLong ranges = new AtomicLong() + def server = HttpServer.create(new InetSocketAddress('127.0.0.1', 0), 0) + server.createContext('/resolve') { exchange -> + redirects.incrementAndGet() + exchange.responseHeaders.add('Location', '/object') + exchange.sendResponseHeaders(302, -1) + exchange.close() + } + server.createContext('/object') { exchange -> + ranges.incrementAndGet() + def match = exchange.requestHeaders.getFirst('Range') =~ /bytes=(\d+)-(\d+)/ + assertTrue(match.matches()) + int start = Integer.parseInt(match.group(1)) + int end = Integer.parseInt(match.group(2)) + exchange.responseHeaders.add('ETag', '"stable"') + exchange.responseHeaders.add('Content-Range', "bytes $start-$end/${bytes.length}") + exchange.sendResponseHeaders(206, end - start + 1) + exchange.responseBody.write(bytes, start, end - start + 1) + exchange.close() + } + server.start() + try { + URL url = new URI("http://127.0.0.1:${server.address.port}/resolve").toURL() + def input = new HttpRangeInputFile(url, new ParquetReadOptions()) + input.newStream().withCloseable { stream -> + def target = java.nio.ByteBuffer.allocate(1024 * 1024) + stream.readFully(target) + assertEquals(37, target.get(0)) + assertEquals(2L, ranges.get()) // Probe plus one 1 MiB bulk request. + stream.seek(0) + stream.readFully(new byte[1024 * 1024]) + assertEquals(2L, ranges.get()) // Default cache retains the bulk range. + def backing = java.nio.ByteBuffer.allocate(1024 * 1024 + 32) + backing.position(13) + backing.limit(1024 * 1024 + 13) + def slice = backing.slice() + slice.position(7) + stream.seek(0) + stream.readFully(slice) + assertEquals(slice.limit(), slice.position()) + assertEquals(0, backing.get(19)) + assertEquals(37, backing.get(20)) + assertEquals(0, backing.array()[backing.limit()]) + assertEquals(2L, ranges.get()) + def direct = java.nio.ByteBuffer.allocateDirect(1024) + stream.seek(0) + stream.readFully(direct) + assertEquals(37, direct.get(0)) + stream.seek(bytes.length) + assertEquals(-1, stream.read(java.nio.ByteBuffer.allocate(1))) + assertEquals(0, stream.read(java.nio.ByteBuffer.allocate(0))) + } + assertEquals(1L, redirects.get()) + def tinyCache = new HttpRangeInputFile(url, new ParquetReadOptions(rangeCacheBytes: 1024)) + tinyCache.newStream().withCloseable { stream -> + long before = ranges.get() + stream.readFully(new byte[1024 * 1024]) + assertEquals(before + 1, ranges.get()) // Request size is independent of cache size. + } + println "HTTP bulk 1 MiB: one data request; redirects=${redirects.get()}, ranges=${ranges.get()}" + assertEquals(2L, redirects.get()) // One resolution per input, never per bulk request. + } finally { server.stop(0) } + } + + @Test + void rangeReadsFallbackAndObjectChange() { + File file = new File(directory, 'large.parquet') + def data = Matrix.builder().columns(id: (0..<20000).toList(), text: (0..<20000).collect { UUID.randomUUID().toString() }).build() + MatrixParquetWriter.builder(data).compressionCodec('UNCOMPRESSED').pageSize(4096).dictionaryEncoding(false).write(file) + byte[] bytes = file.bytes + AtomicLong transferred = new AtomicLong() + boolean changed = false + def server = HttpServer.create(new InetSocketAddress('127.0.0.1', 0), 0) + server.createContext('/file') { exchange -> + String range = exchange.requestHeaders.getFirst('Range') + exchange.responseHeaders.add('ETag', changed ? '"changed"' : '"original"') + if (range != null) { + def match = range =~ /bytes=(\d+)-(\d+)/ + assertTrue(match.matches()) + int start = Integer.parseInt(match.group(1)) + int end = Integer.parseInt(match.group(2)) + exchange.responseHeaders.add('Content-Range', "bytes $start-$end/${bytes.length}") + exchange.sendResponseHeaders(206, end - start + 1) + exchange.responseBody.write(bytes, start, end - start + 1) + transferred.addAndGet(end - start + 1) + } else { + exchange.sendResponseHeaders(200, bytes.length) + exchange.responseBody.write(bytes) + transferred.addAndGet(bytes.length) + } + exchange.close() + } + server.createContext('/download') { exchange -> + exchange.sendResponseHeaders(200, bytes.length) + exchange.responseBody.write(bytes) + exchange.close() + } + server.start() + try { + URL url = new URI("http://127.0.0.1:${server.address.port}/file").toURL() + def selected = MatrixParquetReader.builder().remoteReadMode('RANGE').columns(['id']) + .filter(Predicates.ge('id', 19990)).read(url) + assertEquals((19990..<20000).toList(), selected.column('id')) + println "HTTP bytes full=${bytes.length} transferred=${transferred.get()}" + assertTrue(transferred.get() < bytes.length, "Transferred ${transferred.get()} of ${bytes.length}") + URL download = new URI("http://127.0.0.1:${server.address.port}/download").toURL() + assertEquals(2, MatrixParquetReader.builder().remoteReadMode('AUTO').limit(2).read(download).rowCount()) + assertThrows(IOException) { MatrixParquetReader.builder().remoteReadMode('RANGE').read(download) } + def options = new ParquetReadOptions(remoteReadMode: 'RANGE') + def input = new HttpRangeInputFile(url, options) + changed = true + assertThrows(IOException) { input.newStream().withCloseable { it.read() } } + } finally { server.stop(0) } + } +} diff --git a/matrix-parquet/src/test/groovy/ParquetInteropTest.groovy b/matrix-parquet/src/test/groovy/ParquetInteropTest.groovy new file mode 100644 index 000000000..42d764903 --- /dev/null +++ b/matrix-parquet/src/test/groovy/ParquetInteropTest.groovy @@ -0,0 +1,40 @@ +import static org.junit.jupiter.api.Assertions.* +import static org.junit.jupiter.api.Assumptions.assumeTrue + +import org.junit.jupiter.api.Test + +import se.alipsa.matrix.core.Matrix +import se.alipsa.matrix.parquet.MatrixParquetReader +import se.alipsa.matrix.parquet.MatrixParquetWriter + +class ParquetInteropTest { + @Test + void exportAndReadRewrittenFiles() { + File directory = new File('build/test-interop') + directory.mkdirs() + def data = Matrix.builder('interop').columns(text: ['héllo', null], + payload: [[0, -1] as byte[], [] as byte[]], price: [1.25G, -2.50G], + tags: [['α', null], []]).build() + ['SNAPPY', 'ZSTD'].each { codec -> + File file = new File(directory, "${codec}.parquet") + MatrixParquetWriter.builder(data).compressionCodec(codec).write(file) + File rewritten = new File(directory, "${codec}.cdc.parquet") + if (rewritten.isFile()) { + def result = MatrixParquetReader.read(rewritten) + assertEquals(['héllo', null], result.column('text')) + assertArrayEquals([0, -1] as byte[], (byte[]) result[0, 'payload']) + assertEquals([1.25G, -2.50G], result.column('price')) + assertEquals([['α', null], []], result.column('tags')) + } + } + } + + @Test + void defaultCdcFixtureWhenGenerated() { + File file = new File('build/default-cdc/default-cdc.parquet') + assumeTrue(file.isFile(), 'SKIP default CDC: enable -PrunSlowTests=true with pinned PyArrow 21.0.0') + def data = MatrixParquetReader.read(file) + assertEquals(16384, data.rowCount()) + assertEquals('café-1-' + 'abcd' * 32, data[1, 'text']) + } +} diff --git a/matrix-parquet/src/test/groovy/ParquetLegacyCaptureTest.groovy b/matrix-parquet/src/test/groovy/ParquetLegacyCaptureTest.groovy new file mode 100644 index 000000000..2a660fc5b --- /dev/null +++ b/matrix-parquet/src/test/groovy/ParquetLegacyCaptureTest.groovy @@ -0,0 +1,57 @@ +import static org.junit.jupiter.api.Assertions.* + +import groovy.transform.CompileStatic + +import org.codehaus.groovy.control.CompilerConfiguration +import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfSystemProperty + +import se.alipsa.matrix.core.Matrix + +import java.sql.Timestamp +import java.time.LocalDateTime +import java.time.ZoneId +import java.time.ZoneOffset + +/** Reproduce historical fixtures with the original writer isolated from current classes. */ +@EnabledIfSystemProperty(named = 'parquet.captureLegacy', matches = 'true') +class ParquetLegacyCaptureTest { + private static final String REVISION = 'b2095159664b34ce426685c387adf0bccdb33e01' + + @Test + void captureOriginalWriter() { + def configuration = new CompilerConfiguration() + configuration.addCompilationCustomizers(new ASTTransformationCustomizer(CompileStatic)) + new GroovyClassLoader(getClass().classLoader, configuration).withCloseable { loader -> + List sources = ['ParquetValueWriter', 'MatrixParquetWriter'].collect { name -> + String path = "matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/${name}.groovy" + def process = new ProcessBuilder('git', 'show', "$REVISION:$path").start() + String source = process.inputStream.getText('UTF-8') + assertEquals(0, process.waitFor()) + source + } + Set imports = [] + List bodies = sources.collect { source -> + source.readLines().findAll { line -> + if (line.startsWith('import ')) { imports.add(line); return false } + !line.startsWith('package ') + }.join('\n') + } + loader.parseClass('package se.alipsa.matrix.parquet\n' + imports.join('\n') + '\n' + bodies.join('\n'), 'LegacyWriter.groovy') + Class writer = loader.loadClass('se.alipsa.matrix.parquet.MatrixParquetWriter') + def write = writer.getMethod('write', Matrix, File, boolean) + File root = new File('src/test/resources/optimized') + def dates = Matrix.builder('legacyDates').columns(value: [new Date(1000L), new Timestamp(2000L)]).types(Date).build() + write.invoke(null, dates, new File(root, 'matrix-0.7-date-units.parquet'), true) + def metadata = Matrix.builder('legacyMetadata').columns(amount: [1.25G, -2.50G], + local: [LocalDateTime.of(2024, 1, 1, 0, 0), null]).types(BigDecimal, LocalDateTime).build() + writer.getMethod('write', Matrix, File, ZoneId).invoke(null, metadata, new File(root, 'matrix-0.7-metadata.parquet'), ZoneOffset.UTC) + def failure = assertThrows(java.lang.reflect.InvocationTargetException) { + write.invoke(null, Matrix.builder().columns(value: [[0, -1] as byte[]]).types(byte[]).build(), + new File('build/legacy-invalid-bytes.parquet'), true) + } + assertEquals('org.apache.parquet.schema.InvalidSchemaException', failure.cause.getClass().name) + } + } +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/CountingInput.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/CountingInput.groovy new file mode 100644 index 000000000..d4b82ab0f --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/CountingInput.groovy @@ -0,0 +1,32 @@ +package se.alipsa.matrix.parquet + +import org.apache.parquet.io.InputFile +import org.apache.parquet.io.SeekableInputStream + +import java.nio.ByteBuffer + +/** Test-only exact byte counter over an independent seekable input. */ +class CountingInput implements InputFile { + final InputFile delegate + long totalBytes + CountingInput(InputFile delegate) { this.delegate = delegate } + @Override long getLength() { delegate.length } + @Override SeekableInputStream newStream() { + SeekableInputStream stream = delegate.newStream() + new SeekableInputStream() { + @Override long getPos() { stream.pos } + @Override void seek(long value) { stream.seek(value) } + @Override int read() { int value = stream.read(); if (value >= 0) { totalBytes++ } ; value } + @Override int read(byte[] target, int offset, int length) { + int count = stream.read(target, offset, length) + if (count > 0) { totalBytes += count } + count + } + @Override void readFully(byte[] target) { readFully(target, 0, target.length) } + @Override void readFully(byte[] target, int offset, int length) { stream.readFully(target, offset, length); totalBytes += length } + @Override int read(ByteBuffer target) { int count = stream.read(target); if (count > 0) { totalBytes += count } ; count } + @Override void readFully(ByteBuffer target) { int count = target.remaining(); stream.readFully(target); totalBytes += count } + @Override void close() { stream.close() } + } + } +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/HttpTransportTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/HttpTransportTest.groovy new file mode 100644 index 000000000..d8e3b54d1 --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/HttpTransportTest.groovy @@ -0,0 +1,121 @@ +package se.alipsa.matrix.parquet + +import static org.junit.jupiter.api.Assertions.* + +import org.junit.jupiter.api.Test + +class HttpTransportTest { + @Test + void hubTokenIsOriginScopedAcrossRedirects() { + List requests = [] + URLStreamHandler handler = new URLStreamHandler() { + @Override + protected URLConnection openConnection(URL url) { + def connection = new StubConnection(url, requests.empty ? 302 : 200, + requests.empty ? [Location: 'https://cdn.example/signed'] : [:]) + requests.add(connection) + connection + } + } + URL source = new URL(null, 'https://huggingface.co/datasets/owner/repo/resolve/main/file', handler) + def options = new ParquetReadOptions() + options.hubToken = 'test-secret' + HttpTransport.open(source, options, [:]).disconnect() + assertEquals('Bearer test-secret', requests[0].getRequestProperty('Authorization')) + assertNull(requests[1].getRequestProperty('Authorization')) + assertFalse(options.toMap().containsKey('hubToken')) + } + + @Test + void downgradeAndMalformedRangeFailAndDisconnect() { + StubConnection request + URL source = new URL(null, 'https://huggingface.co/file', new URLStreamHandler() { + @Override protected URLConnection openConnection(URL url) { + request = new StubConnection(url, 302, [Location: 'http://huggingface.co/file']) + request + } + }) + assertThrows(IOException) { HttpTransport.open(source, new ParquetReadOptions(), [:]) } + assertTrue(request.disconnected) + URL malformed = new URL(null, 'https://example.test/file', new URLStreamHandler() { + @Override protected URLConnection openConnection(URL url) { + request = new StubConnection(url, 206, ['Content-Range': 'bytes 0-1/10']) + request + } + }) + assertThrows(IOException) { new HttpRangeInputFile(malformed, new ParquetReadOptions()) } + assertTrue(request.disconnected) + } + + @Test + void truncatedRangeBodyFailsAndDisconnects() { + List requests = [] + URL source = new URL(null, 'https://example.test/file', new URLStreamHandler() { + @Override protected URLConnection openConnection(URL url) { + boolean probe = requests.empty + def connection = new StubConnection(url, 206, ['Content-Range': probe ? 'bytes 0-0/12' : 'bytes 0-3/12']) + requests.add(connection) + connection + } + }) + def input = new HttpRangeInputFile(source, new ParquetReadOptions(rangeCacheBytes: 4)) + assertThrows(EOFException) { input.newStream().withCloseable { it.readFully(new byte[4]) } } + assertTrue(requests.last().disconnected) + assertFalse(requests.first().disconnected) // Fully consumed success allows keep-alive reuse. + } + + @Test + void staticUrlOverloadsUseBoundedTransportAndHubParsing() { + java.time.ZoneId zone = java.time.ZoneId.of('UTC') + [ { URL url -> MatrixParquetReader.read(url) }, + { URL url -> MatrixParquetReader.read(url, zone) }, + { URL url -> MatrixParquetReader.read(url, 'named', zone) } ].each { read -> + StubConnection request + URL url = new URL(null, 'https://example.test/file', new URLStreamHandler() { + @Override protected URLConnection openConnection(URL source) { + request = new StubConnection(source, 302, [Location: 'http://example.test/file']) + request + } + }) + assertThrows(IOException) { read(url) } + assertEquals(10000, request.connectTimeout) + assertEquals(60000, request.readTimeout) + assertTrue(request.disconnected) + } + String invalid = 'hf://datasets/owner/repo/../file.parquet' + assertThrows(IllegalArgumentException) { MatrixParquetReader.readUrl(invalid, zone) } + assertThrows(IllegalArgumentException) { MatrixParquetReader.readUrl(invalid, 'named', zone) } + } + + @Test + void connectionRetriesAreBounded() { + int attempts = 0 + URL source = new URL(null, 'https://example.test/file', new URLStreamHandler() { + @Override protected URLConnection openConnection(URL url) { + attempts++ + new StubConnection(url, 0, [:]) { + @Override int getResponseCode() { throw new IOException('Transient connect failure') } + } + } + }) + assertThrows(IOException) { HttpTransport.open(source, new ParquetReadOptions(), [:]) } + assertEquals(2, attempts) + } + + private static class StubConnection extends HttpURLConnection { + private final int status + private final Map headers + boolean disconnected + StubConnection(URL url, int status, Map headers) { + super(url) + this.status = status + this.headers = headers + } + @Override int getResponseCode() { status } + @Override String getHeaderField(String key) { headers[key] } + @Override void disconnect() { disconnected = true } + @Override boolean usingProxy() { false } + @Override void connect() {} + @Override InputStream getInputStream() { new ByteArrayInputStream([0] as byte[]) } + } +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/LiveHeapSampler.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/LiveHeapSampler.groovy new file mode 100644 index 000000000..506ffa7cb --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/LiveHeapSampler.groovy @@ -0,0 +1,43 @@ +package se.alipsa.matrix.parquet + +import java.lang.management.ManagementFactory + +/** Test-only maximum live-heap sampler using post-GC notifications. */ +class LiveHeapSampler { + final java.util.concurrent.atomic.AtomicLong peak = new java.util.concurrent.atomic.AtomicLong() + private final List emitters = [] + private final Set heapPools = ManagementFactory.memoryPoolMXBeans.findAll { it.type == java.lang.management.MemoryType.HEAP }*.name.toSet() + private final Map initialCounts = [:] + private final java.util.concurrent.CountDownLatch finished = new java.util.concurrent.CountDownLatch(1) + private volatile Map finalCounts + private final javax.management.NotificationListener listener = { notification, handback -> + if (notification.type == com.sun.management.GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION) { + def event = com.sun.management.GarbageCollectionNotificationInfo.from((javax.management.openmbean.CompositeData) notification.userData) + def info = event.gcInfo + if (info.id > (initialCounts[event.gcName] ?: 0L)) { + long used = info.memoryUsageAfterGc.findAll { String pool, usage -> heapPools.contains(pool) }.values().sum { it.used } as long + peak.accumulateAndGet(used, { long a, long b -> Math.max(a, b) } as java.util.function.LongBinaryOperator) + Map target = finalCounts + if (target != null && info.id > (target[event.gcName] ?: 0L)) { finished.countDown() } + } + } + } as javax.management.NotificationListener + + LiveHeapSampler() { + ManagementFactory.garbageCollectorMXBeans.each { collector -> + if (collector instanceof javax.management.NotificationEmitter) { + initialCounts[collector.name] = collector.collectionCount + collector.addNotificationListener(listener, null, null) + emitters.add(collector) + } + } + } + + @SuppressWarnings('ExplicitGarbageCollection') + void finish() { + finalCounts = ManagementFactory.garbageCollectorMXBeans.collectEntries { [(it.name): it.collectionCount] } + System.gc() + assert finished.await(5, java.util.concurrent.TimeUnit.SECONDS) : 'Final GC notification was not observed' + emitters.each { it.removeNotificationListener(listener) } + } +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetDatasetBenchmarkTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetDatasetBenchmarkTest.groovy new file mode 100644 index 000000000..12aa443cf --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetDatasetBenchmarkTest.groovy @@ -0,0 +1,38 @@ +package se.alipsa.matrix.parquet + +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfSystemProperty + +import se.alipsa.matrix.core.Matrix + +/** Opt-in bounded shard-decoding measurements, without timing assertions. */ +@EnabledIfSystemProperty(named = 'parquet.datasetBenchmark', matches = 'true') +class ParquetDatasetBenchmarkTest { + @Test + @SuppressWarnings('ExplicitGarbageCollection') + void compareWorkers() { + File directory = new File('build/dataset-benchmark') + directory.mkdirs() + List files = (0..<4).collect { int shard -> + File file = new File(directory, "shard-${shard}.parquet") + int start = shard * 250000 + MatrixParquetWriter.write(Matrix.builder().columns(id: (start.. + (0..<4).each { run -> + System.gc() + def sampler = new LiveHeapSampler() + long start = System.nanoTime() + def result = MatrixParquetReader.builder().parallelism(workers).maxBatchRows(65536).read(files) + long millis = (System.nanoTime() - start).intdiv(1_000_000) + sampler.finish() + assert result.rowCount() == 1_000_000 + assert result[0, 'id'] == 0 + assert result[999999, 'id'] == 999999 + println "DATASET workers=$workers run=$run rows=${result.rowCount()} millis=$millis peakLiveAfterGc=${sampler.peak.get()}" + } + } + } +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetLayoutBenchmarkTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetLayoutBenchmarkTest.groovy new file mode 100644 index 000000000..7d11a1456 --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetLayoutBenchmarkTest.groovy @@ -0,0 +1,35 @@ +package se.alipsa.matrix.parquet + +import org.apache.parquet.io.LocalInputFile +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfSystemProperty + +/** Opt-in full/projection/filter measurements against externally generated identical layouts. */ +@EnabledIfSystemProperty(named = 'parquet.layoutBenchmark', matches = 'true') +class ParquetLayoutBenchmarkTest { + @Test + @SuppressWarnings('ExplicitGarbageCollection') + void compareLayouts() { + ['ordinary', 'indexed', 'cdc-indexed'].each { layout -> + File file = new File("build/layout-benchmarks/${layout}.parquet") + assert file.isFile() : 'Run src/test/python/generate_benchmarks.py first' + ['full', 'projection', 'selective'].each { mode -> + (0..<4).each { int run -> + System.gc() + def options = new ParquetReadOptions() + if (mode != 'full') { options.columns = ['id'] } + if (mode == 'selective') { options.filter = Predicates.ge('id', 999900L) } + def input = new CountingInput(new LocalInputFile(file.toPath())) + def sampler = new LiveHeapSampler() + long start = System.nanoTime() + def result = MatrixParquetReader.readInput(input, options, null) + long elapsed = System.nanoTime() - start + sampler.finish() + assert result.rowCount() == (mode == 'selective' ? 100 : 1000000) + assert result[result.rowCount() - 1, 'id'] == 999999L + println "LAYOUT layout=$layout mode=$mode run=$run rows=${result.rowCount()} fileBytes=${file.length()} readBytes=${input.totalBytes} millis=${elapsed.intdiv(1000000)} peakLiveAfterGc=${sampler.peak.get()}" + } + } + } + } +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetPruningTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetPruningTest.groovy new file mode 100644 index 000000000..3445c0b74 --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetPruningTest.groovy @@ -0,0 +1,128 @@ +package se.alipsa.matrix.parquet + +import static org.junit.jupiter.api.Assertions.* + +import org.apache.parquet.conf.PlainParquetConfiguration +import org.apache.parquet.hadoop.ParquetFileReader +import org.apache.parquet.io.LocalInputFile +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import se.alipsa.matrix.core.Matrix + + +class ParquetPruningTest { + @TempDir File directory + + @Test + void fixturesHavePhysicalIndexesAndMultiplePages() { + [0, 1].each { cdc -> + [0, 1].each { indexed -> + File file = new File('src/test/resources/optimized', "cdc-${cdc}-index-${indexed}.parquet") + def options = org.apache.parquet.ParquetReadOptions.builder(new PlainParquetConfiguration()).build() + ParquetFileReader.open(new LocalInputFile(file.toPath()), options).withCloseable { reader -> + assertEquals(2, reader.rowGroups.size()) + def column = reader.rowGroups[0].columns.find { it.path.toDotString() == 'id' } + assertEquals(indexed == 1, column.columnIndexReference != null) + assertEquals(indexed == 1, column.offsetIndexReference != null) + if (indexed == 1) { checkIndexes(reader, column, file.length(), reader.rowGroups[0].rowCount) } + } + } + } + } + + private static void checkIndexes(ParquetFileReader reader, column, long length, long rows) { + def offsets = reader.readOffsetIndex(column) + def bounds = reader.readColumnIndex(column) + assertTrue(offsets.pageCount > 1) + assertEquals(offsets.pageCount, bounds.nullPages.size()) + assertEquals(0L, offsets.getFirstRowIndex(0)) + (0.. + assertTrue(offsets.getOffset(page) >= 4) + assertTrue(offsets.getOffset(page) + offsets.getCompressedPageSize(page) <= length) + assertTrue(offsets.getFirstRowIndex(page) < rows) + if (page > 0) { + assertTrue(offsets.getOffset(page) > offsets.getOffset(page - 1)) + assertTrue(offsets.getFirstRowIndex(page) > offsets.getFirstRowIndex(page - 1)) + } + } + } + + @Test + void writerIndexesAcrossOutputPaths() { + def data = Matrix.builder().columns(id: (0..<20000).toList(), text: ['label'] * 20000).build() + File file = new File(directory, 'indexes.parquet') + def writer = MatrixParquetWriter.builder(data).pageSize(1024).dictionaryEncoding(false) + writer.write(file) + def stream = new ByteArrayOutputStream() + writer.write(stream) + [new LocalInputFile(file.toPath()), new InMemoryInputFile(writer.writeBytes()), + new InMemoryInputFile(stream.toByteArray())].each { input -> + def options = org.apache.parquet.ParquetReadOptions.builder(new PlainParquetConfiguration()).build() + ParquetFileReader.open(input, options).withCloseable { reader -> + reader.rowGroups[0].columns.each { column -> checkIndexes(reader, column, input.length, reader.rowGroups[0].rowCount) } + } + } + } + + @Test + void pageIndexesReduceDataReadsWithoutRowGroupPruning() { + File file = new File(directory, 'pages.parquet') + def data = Matrix.builder().columns(id: (0..<20000).toList(), text: ['categorical'] * 20000).build() + MatrixParquetWriter.builder(data).compressionCodec('UNCOMPRESSED').dictionaryEncoding(false) + .pageSize(1024).rowGroupSize(16 * 1024 * 1024L).write(file) + List counts = [] + [false, true].each { enabled -> + def input = new CountingInput(new LocalInputFile(file.toPath())) + def options = new ParquetReadOptions(columns: ['id'], filter: Predicates.ge('id', 19990), useColumnIndexFilter: enabled) + options.useRowGroupFilters = false + def result = MatrixParquetReader.readInput(input, options, null) + assertEquals((19990..<20000).toList(), result.column('id')) + counts.add(input.totalBytes) + } + println "PAGE_INDEX bytes full=${counts[0]} pruned=${counts[1]}" + assertTrue(counts[1] < counts[0], "Indexed totalBytes ${counts[1]} versus full ${counts[0]}") + } + + @Test + void rejectedRowsNeverReachColumnsAndDictionaryStringsAreReused() { + File file = new File(directory, 'dictionary.parquet') + MatrixParquetWriter.write(Matrix.builder().columns(id: [1, 2, 3, 4, 5], text: ['same'] * 5).build(), file) + def result = MatrixParquetReader.read(file) + assertSame(result[0, 'text'], result[4, 'text']) + [Predicates.ge('id', 3), Predicates.ne('id', 3), Predicates.not(Predicates.ge('id', 1))].each { predicate -> + def filtered = MatrixParquetReader.builder().filter(predicate).read(file) + assertEquals(filtered.column('id').size(), filtered.column('text').size()) + assertEquals((1..5).findAll { predicate.test([id: it]) }, filtered.column('id')) + } + def empty = MatrixParquetReader.builder().rowGroups([0]).filter(Predicates.gt('id', 100)).read(file) + assertEquals(['id', 'text'], empty.columnNames()) + assertEquals(0, empty.rowCount()) + } + + @Test + void bloomFilterSkipsAnInteriorAbsentKeyIndependently() { + File file = new File(directory, 'bloom.parquet') + def data = Matrix.builder().columns(id: (0..<10000).collect { 2 * it + 1 }).build() + MatrixParquetWriter.builder(data).compressionCodec('UNCOMPRESSED').dictionaryEncoding(false) + .rowGroupSize(16L * 1024 * 1024).bloomFilterEnabled([id: true]).bloomFilterNDV([id: 10000L]).write(file) + int absent + ParquetFileReader.open(new LocalInputFile(file.toPath()), org.apache.parquet.ParquetReadOptions.builder(new PlainParquetConfiguration()).build()).withCloseable { reader -> + assertEquals(1, reader.rowGroups.size()) + def bloom = reader.readBloomFilter(reader.rowGroups[0].columns[0]) + assertNotNull(bloom) + absent = (2..<19998).find { it % 2 == 0 && !bloom.findHash(bloom.hash(it)) } + } + List counts = [] + [false, true].each { enabled -> + def input = new CountingInput(new LocalInputFile(file.toPath())) + def options = new ParquetReadOptions(filter: Predicates.eq('id', absent), useColumnIndexFilter: false) + options.useRowGroupFilters = enabled + assertEquals(0, MatrixParquetReader.readInput(input, options, null).rowCount()) + counts.add(input.totalBytes) + } + assertTrue(counts[1] < counts[0], "Bloom totalBytes ${counts[1]} versus full ${counts[0]}") + println "BLOOM bytes full=${counts[0]} pruned=${counts[1]}" + } + +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReadBenchmarkTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReadBenchmarkTest.groovy new file mode 100644 index 000000000..4ff865b5b --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReadBenchmarkTest.groovy @@ -0,0 +1,74 @@ +package se.alipsa.matrix.parquet + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path +import org.apache.parquet.example.data.Group +import org.apache.parquet.hadoop.ParquetFileReader +import org.apache.parquet.hadoop.ParquetReader +import org.apache.parquet.hadoop.example.GroupReadSupport +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfSystemProperty + +import se.alipsa.matrix.core.Matrix + +import java.lang.management.ManagementFactory +import java.time.LocalDate + +/** Opt-in comparison against the pre-change Group loop using shared schema conversions. */ +@EnabledIfSystemProperty(named = 'parquet.baseline', matches = 'true') +class ParquetReadBenchmarkTest { + @Test + @SuppressWarnings('ExplicitGarbageCollection') // Opt-in retained-heap benchmark, not production behavior. + void compare() { + int rows = Integer.parseInt(System.getProperty('parquet.benchmarkRows', '1000000')) + ['wide', 'categorical', 'mixed'].each { workload -> + def columns = [id: (0.. + columns["text$c".toString()] = (0.. + (0..<4).each { int run -> + System.gc() + long beforeHeap = ManagementFactory.memoryMXBean.heapMemoryUsage.used + def bean = (com.sun.management.ThreadMXBean) ManagementFactory.threadMXBean + long allocated = bean.getThreadAllocatedBytes(Thread.currentThread().threadId()) + def sampler = new LiveHeapSampler() + long start = System.nanoTime() + Matrix result = columnar ? MatrixParquetReader.read(file) : legacy(file, types) + long elapsed = System.nanoTime() - start + allocated = bean.getThreadAllocatedBytes(Thread.currentThread().threadId()) - allocated + sampler.finish() + long retained = ManagementFactory.memoryMXBean.heapMemoryUsage.used - beforeHeap + assert result.rowCount() == rows + assert result[rows - 1, 'id'] == rows - 1 + println "BENCH workload=$workload columnar=$columnar run=$run rows=$rows bytes=${file.length()} millis=${elapsed.intdiv(1_000_000)} allocated=$allocated retained=$retained peakLiveAfterGc=${sampler.peak.get()}" + } + } + } + } + + private static Matrix legacy(File file, List types) { + Path path = new Path(file.toURI()) + def schema = ParquetFileReader.readFooter(new Configuration(), path).fileMetaData.schema + def names = schema.fields*.name + ParquetReader.builder(new GroupReadSupport(), path).build().withCloseable { reader -> + def builder = Matrix.builder('benchmark').columnNames(names).types(types) + Group row = reader.read() + while (row != null) { + def values = [] + names.eachWithIndex { String name, int index -> values << MatrixParquetReader.readValue(row, name, schema.getType(name), types[index]) } + builder.addRow(values as Object[]) + row = reader.read() + } + builder.build() + } + } +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReviewRegressionTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReviewRegressionTest.groovy new file mode 100644 index 000000000..01d70fb25 --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReviewRegressionTest.groovy @@ -0,0 +1,237 @@ +package se.alipsa.matrix.parquet + +import static org.junit.jupiter.api.Assertions.* + +import org.apache.parquet.conf.PlainParquetConfiguration +import org.apache.parquet.example.data.simple.SimpleGroupFactory +import org.apache.parquet.hadoop.example.ExampleParquetWriter +import org.apache.parquet.io.LocalOutputFile +import org.apache.parquet.schema.MessageTypeParser +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import se.alipsa.matrix.core.Matrix + +import java.time.LocalDateTime +import java.time.ZoneId + +class ParquetReviewRegressionTest { + @TempDir File directory + + @Test + void arrowEmptyDictionaryAndJsonAreReadable() { + File root = new File('src/test/resources/optimized') + def allNull = MatrixParquetReader.read(new File(root, 'arrow-all-null-string.parquet')) + assertEquals(String, allNull.type('s')) + assertEquals([null, null, null], allNull.column('s')) + [null, true, false].each { override -> + def json = MatrixParquetReader.builder().binaryAsString(override).read(new File(root, 'arrow-json.parquet')) + assertEquals(String, json.type('json')) + assertEquals(['{"label":"α"}', null, '{}'], json.column('json')) + } + } + + @Test + void enumIsTextAndBsonRemainsBinary() { + def schema = MessageTypeParser.parseMessageType('message external { optional binary label (ENUM); optional binary document (BSON); }') + File file = new File(directory, 'annotations.parquet') + ExampleParquetWriter.builder(new LocalOutputFile(file.toPath())).withConf(new PlainParquetConfiguration()) + .withType(schema).build().withCloseable { writer -> + writer.write(new SimpleGroupFactory(schema).newGroup().append('label', 'CATEGORY').append('document', 'binary')) + } + def result = MatrixParquetReader.builder().binaryAsString(true).read(file) + assertEquals(String, result.type('label')) + assertEquals('CATEGORY', result[0, 'label']) + assertEquals(byte[], result.type('document')) + assertArrayEquals('binary'.getBytes('UTF-8'), (byte[]) result[0, 'document']) + } + + @Test + void failedDecimalWritesPreserveExistingFileAndCleanTemporaryFiles() { + File file = new File(directory, 'safe.parquet') + MatrixParquetWriter.write(Matrix.builder().columns(d: [1.25G]).build(), file) + boolean posix = java.nio.file.Files.getFileStore(file.toPath()).supportsFileAttributeView('posix') + def permissions = java.nio.file.attribute.PosixFilePermissions.fromString('rw-r-----') + if (posix) { java.nio.file.Files.setPosixFilePermissions(file.toPath(), permissions) } + byte[] original = file.bytes + [2.255G, 123456.00G].each { value -> + assertThrows(IllegalArgumentException) { + MatrixParquetWriter.builder(Matrix.builder().columns(d: [1.00G, value]).build()) + .decimalMeta([d: [5, 2] as int[]]).write(file) + } + assertArrayEquals(original, file.bytes) + assertEquals(['safe.parquet'], directory.list().toList()) + } + MatrixParquetWriter.write(Matrix.builder().columns(d: [2.50G]).build(), file) + assertEquals(2.50G, MatrixParquetReader.read(file)[0, 'd']) + if (posix) { assertEquals(permissions, java.nio.file.Files.getPosixFilePermissions(file.toPath())) } + assertEquals(['safe.parquet'], directory.list().toList()) + } + + @Test + void newFilePermissionsMatchOrdinaryFiles() { + org.junit.jupiter.api.Assumptions.assumeTrue( + java.nio.file.Files.getFileStore(directory.toPath()).supportsFileAttributeView('posix')) + def ordinary = java.nio.file.Files.createFile(new File(directory, 'ordinary').toPath()) + File output = new File(directory, 'new.parquet') + MatrixParquetWriter.write(Matrix.builder().columns(id: [1]).build(), output) + assertEquals(java.nio.file.Files.getPosixFilePermissions(ordinary), + java.nio.file.Files.getPosixFilePermissions(output.toPath())) + } + + @Test + void writingThroughSymbolicLinkPreservesLinkAndUpdatesRealFile() { + org.junit.jupiter.api.Assumptions.assumeTrue( + java.nio.file.Files.getFileStore(directory.toPath()).supportsFileAttributeView('posix')) + File output = new File(directory, 'real.parquet') + MatrixParquetWriter.write(Matrix.builder().columns(d: [1.25G]).build(), output) + def link = new File(directory, 'link.parquet').toPath() + java.nio.file.Files.createSymbolicLink(link, output.toPath().fileName) + byte[] original = output.bytes + assertThrows(IllegalArgumentException) { + MatrixParquetWriter.builder(Matrix.builder().columns(d: [2.255G]).build()) + .decimalMeta([d: [5, 2] as int[]]).write(link.toFile()) + } + assertTrue(java.nio.file.Files.isSymbolicLink(link)) + assertArrayEquals(original, output.bytes) + MatrixParquetWriter.write(Matrix.builder().columns(d: [2.50G]).build(), link.toFile()) + assertTrue(java.nio.file.Files.isSymbolicLink(link)) + assertEquals(output.toPath().fileName, java.nio.file.Files.readSymbolicLink(link)) + assertEquals(2.50G, MatrixParquetReader.read(output)[0, 'd']) + assertEquals(['link.parquet', 'real.parquet'], directory.list().toList().sort()) + } + + @Test + void danglingSymbolicLinksCreateTheirTargetsAndSurviveFailedWrites() { + org.junit.jupiter.api.Assumptions.assumeTrue( + java.nio.file.Files.getFileStore(directory.toPath()).supportsFileAttributeView('posix')) + [false, true].each { absolute -> + File output = new File(directory, "target-${absolute}.parquet") + def link = new File(directory, "dangling-${absolute}.parquet").toPath() + def destination = absolute ? output.toPath().toAbsolutePath() : output.toPath().fileName + java.nio.file.Files.createSymbolicLink(link, destination) + def chain = new File(directory, "chain-${absolute}.parquet").toPath() + java.nio.file.Files.createSymbolicLink(chain, link.fileName) + assertFalse(output.exists()) + assertThrows(IllegalArgumentException) { + MatrixParquetWriter.builder(Matrix.builder().columns(d: [2.255G]).build()) + .decimalMeta([d: [5, 2] as int[]]).write(chain.toFile()) + } + assertFalse(output.exists()) + assertTrue(java.nio.file.Files.isSymbolicLink(link)) + assertTrue(java.nio.file.Files.isSymbolicLink(chain)) + MatrixParquetWriter.write(Matrix.builder().columns(d: [2.50G]).build(), chain.toFile()) + assertEquals(destination, java.nio.file.Files.readSymbolicLink(link)) + assertEquals(link.fileName, java.nio.file.Files.readSymbolicLink(chain)) + assertEquals(2.50G, MatrixParquetReader.read(output)[0, 'd']) + } + assertFalse(directory.list().any { it.startsWith('.matrix-parquet-') }) + } + + @Test + void cyclicSymbolicLinksFailWithoutReplacingLinks() { + org.junit.jupiter.api.Assumptions.assumeTrue( + java.nio.file.Files.getFileStore(directory.toPath()).supportsFileAttributeView('posix')) + def first = new File(directory, 'first.parquet').toPath() + def second = new File(directory, 'second.parquet').toPath() + java.nio.file.Files.createSymbolicLink(first, second.fileName) + java.nio.file.Files.createSymbolicLink(second, first.fileName) + assertThrows(java.nio.file.FileSystemLoopException) { + MatrixParquetWriter.write(Matrix.builder().columns(id: [1]).build(), first.toFile()) + } + assertTrue(java.nio.file.Files.isSymbolicLink(first)) + assertTrue(java.nio.file.Files.isSymbolicLink(second)) + } + + @Test + void nullMapOptionsPreserveDefaultsAndRejectNullBloomEntries() { + def read = ParquetReadOptions.fromMap([useColumnIndexFilter: null, binaryAsString: null]) + assertTrue(read.useColumnIndexFilter) + assertNull(read.binaryAsString) + def write = ParquetWriteOptions.fromMap([dictionaryEncoding: null, sizeStatisticsEnabled: null, + bloomFilterEnabled: null, bloomFilterNDV: null, writerVersion: null, pageSize: null]) + assertTrue(write.dictionaryEncoding) + assertTrue(write.sizeStatisticsEnabled) + assertEquals([:], write.bloomFilterEnabled) + assertEquals('V1', write.writerVersion) + assertEquals(1048576, write.pageSize) + assertThrows(IllegalArgumentException) { ParquetWriteOptions.fromMap([bloomFilterEnabled: [id: null]]) } + } + + @Test + void localSourcesAndCompatibleIndexesAgreeAcrossEntryPoints() { + File file = new File(directory, 'indexed.parquet') + def data = Matrix.builder().columns(id: [1, 2], when: [LocalDateTime.of(2024, 1, 1, 0, 0)] * 2).build() + data.createIndex('id') + MatrixParquetWriter.builder(data).zoneId('UTC').write(file) + ZoneId zone = ZoneId.of('Europe/Stockholm') + [MatrixParquetReader.read(file, zone), MatrixParquetReader.read(directory, zone), + MatrixParquetReader.read(directory, 'named', zone), MatrixParquetReader.read(directory.toPath(), zone), + MatrixParquetReader.read(directory, 'named')].each { result -> + assertTrue(result.hasIndex()) + assertEquals(['id'], result.indexedColumns()) + } + assertEquals(LocalDateTime.of(2024, 1, 1, 1, 0), MatrixParquetReader.read(directory, zone)[0, 'when']) + [file, directory].each { source -> + def result = MatrixParquetReader.builder().sourceFileColumn('_source').read(source) + assertEquals([file.canonicalPath] * 2, result.column('_source')) + assertTrue(result.hasIndex()) + MatrixParquetReader.builder().sourceFileColumn('_source').batches(source).withCloseable { batches -> + assertEquals([file.canonicalPath] * 2, batches.next().column('_source')) + } + } + MatrixParquetReader.builder().limit(1).batches([file]).withCloseable { batches -> + def limited = batches.next() + assertEquals(1, limited.rowCount()) + assertTrue(limited.hasIndex()) + } + assertThrows(IllegalArgumentException) { MatrixParquetReader.builder().sourceFileColumn('id').read(file) } + } + + @Test + void shardBatchesAndCollectedReadsUseTheCommonIndexPolicy() { + List files = [] + ['id', 'text'].eachWithIndex { indexColumn, int number -> + File file = new File(directory, "shard-${number}.parquet") + def matrix = Matrix.builder().columns(id: [1, 2], text: ['a', 'b']).build() + matrix.createIndex(indexColumn) + MatrixParquetWriter.write(matrix, file) + files.add(file) + } + assertFalse(MatrixParquetReader.read(files).hasIndex()) + MatrixParquetReader.builder().batches(files).withCloseable { batches -> + while (batches.hasNext()) { assertFalse(batches.next().hasIndex()) } + } + assertFalse(MatrixParquetReader.builder().columns(['text']).read(files[0]).hasIndex()) + } + + @Test + void workerThreadsAreDaemonAndNestedErrorsIncludeTheirPath() { + File file = new File(directory, 'workers.parquet') + MatrixParquetWriter.write(Matrix.builder().columns(id: [1, 2]).build(), file) + MatrixParquetReader.builder().batches([file]).withCloseable { batches -> + assertTrue(batches.hasNext()) + def workers = Thread.allStackTraces.keySet().findAll { it.name == 'matrix-parquet-dataset' } + assertFalse(workers.empty) + assertTrue(workers.every { it.daemon }) + } + def invalid = Matrix.builder().columns(items: [[[value: [1, 'mixed']]]]).build() + def failure = assertThrows(IllegalArgumentException) { MatrixParquetWriter.writeBytes(invalid) } + assertTrue(failure.message.contains('items.element.value.element'), failure.message) + } + + @Test + void projectedLegacyBinaryFailsOnlyWhenRequired() { + def schema = MessageTypeParser.parseMessageType('message legacy { optional int32 id; optional binary payload; }') + File file = new File(directory, 'legacy.parquet') + ExampleParquetWriter.builder(new LocalOutputFile(file.toPath())).withConf(new PlainParquetConfiguration()) + .withType(schema).withExtraMetaData(['matrix.columnTypes': 'java.lang.Integer,[B']).build().withCloseable { writer -> + writer.write(new SimpleGroupFactory(schema).newGroup().append('id', 1).append('payload', '[B@lost')) + } + assertEquals([1], MatrixParquetReader.builder().columns(['id']).read(file).column('id')) + assertThrows(IllegalArgumentException) { MatrixParquetReader.read(file) } + assertThrows(IllegalArgumentException) { + MatrixParquetReader.builder().columns(['id']).filter(Predicates.isNull('payload')).read(file) + } + } +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetTypePolicyTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetTypePolicyTest.groovy new file mode 100644 index 000000000..b7f9824f5 --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetTypePolicyTest.groovy @@ -0,0 +1,247 @@ +package se.alipsa.matrix.parquet + +import static org.junit.jupiter.api.Assertions.* + +import org.apache.parquet.conf.PlainParquetConfiguration +import org.apache.parquet.example.data.simple.SimpleGroupFactory +import org.apache.parquet.hadoop.example.ExampleParquetWriter +import org.apache.parquet.io.LocalOutputFile +import org.apache.parquet.io.api.Binary +import org.apache.parquet.schema.LogicalTypeAnnotation as Logical +import org.apache.parquet.schema.MessageTypeParser +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName as Physical +import org.apache.parquet.schema.Types +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import se.alipsa.matrix.core.Matrix + +import java.sql.Timestamp +import java.time.* +import java.time.temporal.Temporal + +class ParquetTypePolicyTest { + @TempDir File directory + + private enum Category { + FIRST + @Override String toString() { 'display label' } + } + + @Test + void everyWriterMetadataNameIsAllowlistedAndCanonical() { + Map examples = [(String): 'hello', (Integer): 1, (Long): 1L, (Float): 1F, + (Double): 1D, (Boolean): true, (Byte): (byte) 1, (Short): (short) 1, + (BigInteger): BigInteger.TEN, (BigDecimal): 1.25G, (Character): 'x' as char, + (byte[]): [0, -1] as byte[], (LocalDate): LocalDate.of(2024, 1, 1), + (LocalDateTime): LocalDateTime.of(2024, 1, 1, 0, 0), (LocalTime): LocalTime.of(1, 2, 3, 456789123), + (Instant): Instant.parse('2024-01-01T00:00:00.123456Z'), (UUID): UUID.randomUUID(), + (Date): new Date(0), (java.sql.Date): java.sql.Date.valueOf('2024-01-01'), + (java.sql.Time): java.sql.Time.valueOf('12:00:00'), (Timestamp): Timestamp.from(Instant.EPOCH), + (List): ['label', null], (Map): [label: 'value'], (Category): Category.FIRST] + examples.each { Class declaration, Object value -> + def matrix = Matrix.builder('types').columns(value: [value, null]).types(declaration).build() + byte[] bytes = MatrixParquetWriter.writeBytes(matrix) + File file = new File(directory, declaration.simpleName + '.parquet') + def spi = new ParquetFormatProvider() + spi.write(matrix, file, [:]) + def stream = new ByteArrayOutputStream() + MatrixParquetWriter.builder(matrix).write(stream) + [MatrixParquetReader.read(bytes), spi.read(file, [:]), + MatrixParquetReader.read(new ByteArrayInputStream(stream.toByteArray()))].each { result -> + assertTrue(ColumnTypePolicy.METADATA_TYPES.containsKey(result.type('value').name)) + if (declaration == Category) { assertEquals('FIRST', result[0, 'value']) } + else if (declaration == Character) { assertEquals('x', result[0, 'value']) } + else if (declaration == byte[]) { assertArrayEquals((byte[]) value, (byte[]) result[0, 'value']) } + else { assertEquals(value, result[0, 'value'], "Round trip $declaration") } + assertNull(result[1, 'value']) + } + } + } + + @Test + void supertypesAllNullAndConcreteMismatch() { + [(Number): 1, (CharSequence): 'a', (Comparable): 1, (Serializable): 1, + (Temporal): LocalDate.of(2024, 1, 1), (Object): 1.25G].each { Class type, Object value -> + def result = MatrixParquetReader.read(MatrixParquetWriter.writeBytes( + Matrix.builder().columns(value: [value, null]).types(type).build())) + assertEquals(value.getClass(), result.type('value')) + } + [Number, Temporal].each { Class type -> + assertThrows(IllegalArgumentException) { MatrixParquetWriter.writeBytes(Matrix.builder().columns(value: [null]).types(type).build()) } + } + def empty = MatrixParquetReader.read(MatrixParquetWriter.writeBytes(Matrix.builder().columns(value: [null]).build())) + assertEquals(String, empty.type('value')) + [ZonedDateTime.now(), OffsetDateTime.now(), new Object()].each { value -> + assertThrows(IllegalArgumentException) { MatrixParquetWriter.writeBytes(Matrix.builder().columns(value: [value]).build()) } + } + [Instant.MIN, Instant.MAX, Instant.EPOCH.plusNanos(1)].each { instant -> + assertThrows(IllegalArgumentException) { MatrixParquetWriter.writeBytes(Matrix.builder().columns(value: [instant]).build()) } + } + } + + @Test + void temporalSubclassPoliciesAreValidatedBeforeOutput() { + File target = new File(directory, 'dates.parquet') + target.text = 'preserve' + [[new Date(1000), new Timestamp(2000)], [new java.sql.Date(1000), new Timestamp(2000)]].each { pair -> + [pair, pair.reverse()].each { values -> + def failure = assertThrows(IllegalArgumentException) { + MatrixParquetWriter.write(Matrix.builder().columns(value: values).types(Date).build(), target) + } + assertTrue(failure.message.contains('row 1')) + assertEquals('preserve', target.text) + } + } + [new Timestamp(2000), java.sql.Date.valueOf('2024-01-01'), java.sql.Time.valueOf('12:00:00')].each { value -> + def result = MatrixParquetReader.read(MatrixParquetWriter.writeBytes( + Matrix.builder().columns(value: [value]).types(Date).build())) + assertEquals(Date, result.type('value')) + assertEquals(value.time, result[0, 'value'].time) + } + } + + @Test + void decimalsUseNestedAndUniformSettingsWithoutDoubleFallback() { + def matrix = Matrix.builder().columns(price: [1.5G, 2.25G], nested: [[[amount: 1.25G]], [[amount: 2.75G]]]).types(Number, List).build() + def options = new ParquetWriteOptions().inferPrecisionAndScale(false).precision(8).scale(2) + .decimalMeta(['nested.element.amount': [6, 3] as int[]]) + def result = MatrixParquetReader.read(MatrixParquetWriter.writeBytes(matrix, options)) + assertEquals(BigDecimal, result.type('price')) + assertEquals(1.50G, result[0, 'price']) + assertEquals([[amount: 1.250G]], result[0, 'nested']) + assertThrows(IllegalArgumentException) { MatrixParquetWriter.writeBytes(matrix, new ParquetWriteOptions().inferPrecisionAndScale(false)) } + assertThrows(IllegalArgumentException) { MatrixParquetWriter.writeBytes(matrix, new ParquetWriteOptions().decimalMeta([missing: [6, 2] as int[]])) } + def precise = 12345678901234567890.123456789012345678G + def exact = MatrixParquetReader.read(MatrixParquetWriter.writeBytes(Matrix.builder().columns(value: [precise]).build())) + assertEquals(precise, exact[0, 'value']) + def allNull = MatrixParquetReader.read(MatrixParquetWriter.writeBytes(Matrix.builder().columns(value: [null]).types(BigDecimal).build())) + assertEquals(BigDecimal, allNull.type('value')) + assertNull(allNull[0, 'value']) + def negative = Matrix.builder().columns(value: [new BigDecimal('1E+9')]).build() + assertEquals(new BigDecimal('1000000000'), MatrixParquetReader.read(MatrixParquetWriter.writeBytes(negative))[0, 'value']) + } + + @Test + void externalIntegerDecimalsAndTimestampUnits() { + def builder = Types.buildMessage() + .optional(Physical.INT32).as(Logical.decimalType(2, 9)).named('small') + .optional(Physical.INT64).as(Logical.decimalType(4, 18)).named('large') + Logical.TimeUnit.values().each { unit -> + [false, true].each { adjusted -> builder.optional(Physical.INT64).as(Logical.timestampType(adjusted, unit)).named("${unit}_${adjusted}") } + } + def schema = builder.named('external') + File file = new File(directory, 'external.parquet') + ExampleParquetWriter.builder(new LocalOutputFile(file.toPath())).withConf(new PlainParquetConfiguration()).withType(schema).build().withCloseable { writer -> + def group = new SimpleGroupFactory(schema).newGroup().append('small', -123).append('large', -123456L) + schema.fields.drop(2).each { field -> group.append(field.name, -1L) } + writer.write(group) + } + def result = MatrixParquetReader.builder().zoneId('Europe/Stockholm').read(file) + assertEquals(-1.23G, result[0, 'small']) + assertEquals(-12.3456G, result[0, 'large']) + Logical.TimeUnit.values().each { unit -> + long nanos = unit == Logical.TimeUnit.MILLIS ? 1_000_000L : unit == Logical.TimeUnit.MICROS ? 1000L : 1L + Instant instant = Instant.EPOCH.minusNanos(nanos) + assertEquals(LocalDateTime.ofInstant(instant, ZoneOffset.UTC), result[0, "${unit}_false".toString()]) + assertEquals(LocalDateTime.ofInstant(instant, ZoneId.of('Europe/Stockholm')), result[0, "${unit}_true".toString()]) + } + } + + @Test + void explicitUnsupportedTypesAndUntrustedMetadata() { + ['optional int96 value;', 'optional fixed_len_byte_array(2) value (FLOAT16);'].eachWithIndex { declaration, index -> + def schema = MessageTypeParser.parseMessageType("message external { $declaration }") + File file = new File(directory, "unsupported${index}.parquet") + ExampleParquetWriter.builder(new LocalOutputFile(file.toPath())).withConf(new PlainParquetConfiguration()).withType(schema).build().withCloseable { writer -> + def group = new SimpleGroupFactory(schema).newGroup() + group.add('value', Binary.fromConstantByteArray(new byte[index == 0 ? 12 : 2])) + writer.write(group) + } + assertThrows(IllegalArgumentException) { MatrixParquetReader.read(file) } + } + def schema = Types.buildMessage().optional(Physical.INT32).named('value').named('unknown') + File file = new File(directory, 'unknown.parquet') + ExampleParquetWriter.builder(new LocalOutputFile(file.toPath())).withConf(new PlainParquetConfiguration()).withType(schema) + .withExtraMetaData(['matrix.columnTypes': 'does.not.Exist']).build().withCloseable { writer -> writer.write(new SimpleGroupFactory(schema).newGroup().append('value', 42)) } + assertEquals(Integer, MatrixParquetReader.read(file).type('value')) + } + + @Test + void knownButIncompatibleMetadataFallsBackToSchema() { + def schema = MessageTypeParser.parseMessageType('message external { optional binary payload; optional group record { optional int32 id; } }') + File file = new File(directory, 'incompatible-metadata.parquet') + ExampleParquetWriter.builder(new LocalOutputFile(file.toPath())).withConf(new PlainParquetConfiguration()).withType(schema) + .withExtraMetaData(['matrix.columnTypes': 'java.util.UUID,java.util.List', 'matrix.parquet.version': '2']) + .build().withCloseable { writer -> + def row = new SimpleGroupFactory(schema).newGroup() + row.add('payload', Binary.fromConstantByteArray([0, -1] as byte[])) + row.addGroup('record').append('id', 42) + writer.write(row) + } + def result = MatrixParquetReader.read(file) + assertEquals(byte[], result.type('payload')) + assertArrayEquals([0, -1] as byte[], (byte[]) result[0, 'payload']) + assertEquals(Map, result.type('record')) + assertEquals([id: 42], result[0, 'record']) + } + + @Test + void legacyListLayoutsAndNestedValidation() { + def primitive = MessageTypeParser.parseMessageType('message legacy { optional group tags (LIST) { repeated binary array (STRING); } }') + File file = new File(directory, 'legacy-list.parquet') + ExampleParquetWriter.builder(new LocalOutputFile(file.toPath())).withConf(new PlainParquetConfiguration()).withType(primitive).build().withCloseable { writer -> + def row = new SimpleGroupFactory(primitive).newGroup() + row.addGroup('tags').append('array', 'α').append('array', 'β') + writer.write(row) + } + assertEquals(['α', 'β'], MatrixParquetReader.read(file)[0, 'tags']) + def structs = MessageTypeParser.parseMessageType('message legacy { optional group tags (LIST) { repeated group array { optional binary text (STRING); } } }') + File structured = new File(directory, 'legacy-struct-list.parquet') + ExampleParquetWriter.builder(new LocalOutputFile(structured.toPath())).withConf(new PlainParquetConfiguration()).withType(structs).build().withCloseable { writer -> + def row = new SimpleGroupFactory(structs).newGroup() + row.addGroup('tags').addGroup('array').append('text', 'α') + writer.write(row) + } + assertEquals([[text: 'α']], MatrixParquetReader.read(structured)[0, 'tags']) + [Matrix.builder().columns(value: [[[1], 'mixed']]).build(), + Matrix.builder().columns(value: [[(null): 'missing key']]).build()].each { invalid -> + File target = new File(directory, 'preserved.parquet') + target.text = 'preserve' + assertThrows(IllegalArgumentException) { MatrixParquetWriter.write(invalid, target) } + assertEquals('preserve', target.text) + } + } + + @Test + void utf8OrderingAndLegacyBinaryPredicatesMatchPushdown() { + def values = ['a', '\uE000', '\uD800\uDC00'] + File file = new File(directory, 'unicode.parquet') + MatrixParquetWriter.builder(Matrix.builder().columns(text: values).build()) + .pageSize(64).dictionaryEncoding(false).write(file) + [true, false].each { indexed -> + def result = MatrixParquetReader.builder().useColumnIndexFilter(indexed) + .filter(Predicates.lt('text', '\uD800\uDC00')).read(file) + assertEquals(['a', '\uE000'], result.column('text')) + } + File legacy = new File('src/test/resources/optimized/matrix-0.7-strings.parquet') + def selected = MatrixParquetReader.builder().filter(Predicates.eq('text', 'héllo'.getBytes('UTF-8'))).read(legacy) + assertEquals(['héllo'], selected.column('text')) + } + + @Test + void logicalPredicatesAndPrecisionValidation() { + def matrix = Matrix.builder().columns(price: [1.25G, 2.75G, null], + instant: [Instant.EPOCH, Instant.EPOCH.plusSeconds(1), null], + time: [LocalTime.NOON, LocalTime.NOON.plusNanos(123456789), null]).build() + File file = new File(directory, 'predicates.parquet') + MatrixParquetWriter.write(matrix, file) + [Predicates.ge('price', 2.00G), Predicates.ge('instant', Instant.EPOCH.plusSeconds(1)), + Predicates.gt('time', LocalTime.NOON), Predicates.isNull('price')].each { filter -> + def result = MatrixParquetReader.builder().columns(['price']).filter(filter).read(file) + assertEquals(1, result.rowCount()) + } + assertThrows(ArithmeticException) { MatrixParquetReader.builder().filter(Predicates.eq('price', 1.251G)).read(file) } + assertThrows(IllegalArgumentException) { MatrixParquetReader.builder().filter(Predicates.eq('instant', Instant.EPOCH.plusNanos(1))).read(file) } + } +} diff --git a/matrix-parquet/src/test/python/generate_benchmarks.py b/matrix-parquet/src/test/python/generate_benchmarks.py new file mode 100644 index 000000000..086b799f5 --- /dev/null +++ b/matrix-parquet/src/test/python/generate_benchmarks.py @@ -0,0 +1,19 @@ +"""Optional million-row layout benchmark inputs; never checked into Git.""" +from pathlib import Path +import pyarrow as pa +import pyarrow.parquet as pq + +assert pa.__version__ == '21.0.0' +out = Path(__file__).parents[3] / 'build/layout-benchmarks' +out.mkdir(parents=True, exist_ok=True) +rows = 1_000_000 +table = pa.table({'id': pa.array(range(rows), type=pa.int64()), **{ + f'text{c}': [f'café-{c}-{i % 1000}-' + 'abcd' * 32 for i in range(rows)] + for c in range(6)}}) +for name, cdc, index in [('ordinary', False, False), ('indexed', False, True), ('cdc-indexed', True, True)]: + target = out / f'{name}.parquet' + pq.write_table(table, target, compression='snappy', row_group_size=250_000, + data_page_size=64 * 1024, write_page_index=index, + use_content_defined_chunking=cdc) + assert pq.read_table(target).equals(table) + print(f'{name}: rows={rows}, bytes={target.stat().st_size}') diff --git a/matrix-parquet/src/test/python/generate_hub_urls.py b/matrix-parquet/src/test/python/generate_hub_urls.py new file mode 100644 index 000000000..e2b8b5d6b --- /dev/null +++ b/matrix-parquet/src/test/python/generate_hub_urls.py @@ -0,0 +1,25 @@ +"""Regenerate offline reference URL cases with huggingface_hub==2.2.0 (Python >=3.10).""" +from pathlib import Path +import huggingface_hub +from huggingface_hub import parse_hf_uri + +assert huggingface_hub.__version__ == '2.2.0' +cases = [ + 'hf://datasets/owner/repo/train.parquet', + 'hf://datasets/owner/repo@v1/train.parquet', + 'hf://datasets/owner/repo@feature%2Fbranch/train.parquet', + 'hf://datasets/owner/repo@refs/convert/parquet/train.parquet', + 'hf://datasets/owner/repo@refs%2Fconvert%2Fparquet/train.parquet', + 'hf://datasets/owner/repo@refs/pr/10/train.parquet', + 'hf://datasets/owner/repo@refs/convert/parquet-v2/train.parquet', + 'hf://datasets/owner/repo/sub/file@1.parquet', + 'hf://datasets/owner/repo@v%2B1/café x.parquet', + 'hf://datasets/owner/repo/file%20x.parquet', + 'hf://datasets/owner/repo@v*/file*.parquet', + 'hf://datasets/owner/repo@feature%252Fbranch/a%2Fb.parquet', +] +output = Path(__file__).parents[1] / 'resources/optimized/hub-urls.tsv' +output.write_text('# huggingface_hub=2.2.0\n' + ''.join( + uri + '\t' + parse_hf_uri(uri).to_url().replace('/blob/', '/resolve/', 1) + '\n' + for uri in cases), encoding='utf-8') +print(f'Generated {len(cases)} offline Hub client reference cases') diff --git a/matrix-parquet/src/test/python/generate_optimized.py b/matrix-parquet/src/test/python/generate_optimized.py new file mode 100644 index 000000000..39c455a35 --- /dev/null +++ b/matrix-parquet/src/test/python/generate_optimized.py @@ -0,0 +1,71 @@ +"""Reproduce fixtures: python -m pip install pyarrow==21.0.0; python generate_optimized.py [output].""" +import hashlib +import json +import sys +from datetime import datetime, date +from decimal import Decimal +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + +assert pa.__version__ == '21.0.0', pa.__version__ +out = Path(sys.argv[1]) if len(sys.argv) > 1 else Path(__file__).parents[1] / 'resources/optimized' +out.mkdir(parents=True, exist_ok=True) +rows = 4096 +table = pa.table({ + 'id': pa.array(range(rows), type=pa.int64()), + 'text': [None if i % 17 == 0 else f'café-{i % 31}-' + 'abcd' * 32 for i in range(rows)], + 'payload': pa.array([bytes([i % 256, 0, 255]) for i in range(rows)], type=pa.binary()), + 'price': pa.array([Decimal(i).scaleb(-2) for i in range(rows)], type=pa.decimal128(12, 2)), + 'local': pa.array([datetime(2024, 3, 31, 1, 30)] * rows, type=pa.timestamp('us')), + 'date': pa.array([date(2024, 1, 1)] * rows), + 'nulls': pa.array([None] * rows, type=pa.string()), + 'media': pa.array([None if i % 11 == 0 else {'bytes': bytes([i % 256, 0, 255]), 'path': f'image-{i % 31}.jpg'} for i in range(rows)], type=pa.struct([('bytes', pa.binary()), ('path', pa.string())])), + 'tags': pa.array([None if i % 19 == 0 else ([] if i % 7 == 0 else ['α', None, 'β']) for i in range(rows)], type=pa.list_(pa.string())), +}) +manifest = {'pyarrow': pa.__version__, 'rows': rows, 'expected': {'first_id': 0, 'last_id': rows - 1, 'price_123': '1.23', 'local': '2024-03-31T01:30:00', 'payload_0_hex': '0000ff', 'tags_1': ['α', None, 'β']}, 'files': {}} +# Eight-case binary orthogonal array: each pair of CDC/index/codec/dictionary/page factors +# covers all four combinations, rather than requiring the 32-case Cartesian product. +for cdc in (False, True): + for index in (False, True): + for alternate in (False, True): + suffix = '-alternate' if alternate else '' + name = f'cdc-{int(cdc)}-index-{int(index)}{suffix}.parquet' + options = dict(compression='zstd' if cdc ^ alternate else 'snappy', + use_dictionary=['text', 'tags.list.element'] if index ^ alternate else False, + data_page_version='2.0' if cdc ^ index ^ alternate else '1.0', + write_page_index=index, row_group_size=2048, + data_page_size=4096, write_batch_size=128, dictionary_pagesize_limit=1024, + use_content_defined_chunking=dict(min_chunk_size=4096, max_chunk_size=16384, norm_level=0) if cdc else False) + pq.write_table(table, out / name, **options) + assert pq.read_table(out / name).equals(table) + assert (out / name).stat().st_size <= 1024 * 1024 + manifest['files'][name] = {'options': options, 'sha256': hashlib.sha256((out / name).read_bytes()).hexdigest(), 'schema': str(table.schema)} +# Arrow emits an empty dictionary for an all-null string column. +for name, extra in { + 'arrow-all-null-string.parquet': pa.table({'s': pa.array([None] * 3, type=pa.string())}), + 'arrow-json.parquet': pa.table({'json': pa.array(['{"label":"α"}', None, '{}'], type=pa.json_())}), +}.items(): + pq.write_table(extra, out / name, use_dictionary=True, write_page_index=True) + manifest['files'][name] = {'sha256': hashlib.sha256((out / name).read_bytes()).hexdigest(), + 'rows': extra.num_rows, 'schema': str(extra.schema)} +# A synthetic legacy [B footer exercises contextual rejection; the original writer cannot +# produce a valid byte[] file in this checkout (InvalidSchemaException, captured separately). +legacy_binary = pa.table({'payload': pa.array([b'[B@legacy-text'], type=pa.binary())}) +legacy_binary = legacy_binary.replace_schema_metadata({b'matrix.columnTypes': b'[B'}) +pq.write_table(legacy_binary, out / 'legacy-binary-metadata.parquet') +manifest['files']['legacy-binary-metadata.parquet'] = { + 'provenance': 'synthetic legacy [B metadata, not claimed as original recoverable bytes', + 'sha256': hashlib.sha256((out / 'legacy-binary-metadata.parquet').read_bytes()).hexdigest(), + 'expected': 'unrecoverable-legacy-binary error'} +for name in ('matrix-0.7-strings.parquet', 'matrix-0.7-date-units.parquet', 'matrix-0.7-metadata.parquet'): + if (out / name).is_file(): + manifest['files'][name] = { + 'provenance': 'original writer at b2095159664b34ce426685c387adf0bccdb33e01', + 'sha256': hashlib.sha256((out / name).read_bytes()).hexdigest(), + 'schema': str(pq.read_schema(out / name))} +if '--default-cdc' in sys.argv: + pq.write_table(pa.concat_tables([table] * 4), out / 'default-cdc.parquet', use_content_defined_chunking=True, write_page_index=True, use_dictionary=False) +assert sum(p.stat().st_size for p in out.glob('*.parquet')) <= 4 * 1024 * 1024 +(out / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n') diff --git a/matrix-parquet/src/test/python/verify_matrix_output.py b/matrix-parquet/src/test/python/verify_matrix_output.py new file mode 100644 index 000000000..ca6c706be --- /dev/null +++ b/matrix-parquet/src/test/python/verify_matrix_output.py @@ -0,0 +1,21 @@ +"""Run after :matrix-parquet:test, then rerun ParquetInteropTest to read CDC rewrites in Java.""" +from decimal import Decimal +from pathlib import Path +import pyarrow as pa +import pyarrow.parquet as pq + +assert pa.__version__ == '21.0.0' +root = Path(__file__).parents[3] / 'build/test-interop' +for codec in ('SNAPPY', 'ZSTD'): + table = pq.read_table(root / f'{codec}.parquet') + assert pa.types.is_string(table.schema.field('text').type) + assert pa.types.is_binary(table.schema.field('payload').type) + assert table['text'].to_pylist() == ['héllo', None] + assert table['payload'].to_pylist() == [bytes([0, 255]), b''] + assert table['price'].to_pylist() == [Decimal('1.25'), Decimal('-2.50')] + assert table['tags'].to_pylist() == [['α', None], []] + assert table.schema.metadata[b'matrix.parquet.version'] == b'2' + output = root / f'{codec}.cdc.parquet' + pq.write_table(table, output, compression=codec.lower(), use_content_defined_chunking=True, write_page_index=True) + assert pq.read_table(output).equals(table, check_metadata=True) + print(f'{codec}: annotated strings, binary and exact decimals verified; CDC rewrite preserves metadata') diff --git a/matrix-parquet/src/test/resources/optimized/.gitignore b/matrix-parquet/src/test/resources/optimized/.gitignore new file mode 100644 index 000000000..0b7029b7e --- /dev/null +++ b/matrix-parquet/src/test/resources/optimized/.gitignore @@ -0,0 +1,2 @@ +# Original Hadoop writer capture creates local checksum sidecars. +.*.crc diff --git a/matrix-parquet/src/test/resources/optimized/README.md b/matrix-parquet/src/test/resources/optimized/README.md new file mode 100644 index 000000000..65bfc6ae8 --- /dev/null +++ b/matrix-parquet/src/test/resources/optimized/README.md @@ -0,0 +1,45 @@ +# Interoperability fixture provenance + +The manifest records Arrow 21.0.0 schemas, write options, expected values and SHA-256 hashes. +Eight cases cover CDC/index/codec/dictionary/page-version pairs, with two row groups, raw media +bytes, local timestamps, exact decimals, nulls and nested values. CDC uses 4–16 KiB chunks to +keep checked-in files below 1 MiB each and 4 MiB in total. Java tests inspect actual page indexes. + +Recreate the Arrow cases from the repository root: + +```bash +python3 -m venv /tmp/matrix-parquet-fixtures +/tmp/matrix-parquet-fixtures/bin/python -m pip install pyarrow==21.0.0 +/tmp/matrix-parquet-fixtures/bin/python matrix-parquet/src/test/python/generate_optimized.py +PARQUET_PYTHON=/tmp/matrix-parquet-fixtures/bin/python \ + ./gradlew :matrix-parquet:test -PrunSlowTests=true +``` + +The slow command also generates a default-size CDC file under build/default-cdc. Python is not +required for ordinary tests, which consume the checked-in fixtures. CI installs pinned Python +dependencies and regenerates both kinds. + +`matrix-0.7-strings.parquet`, `matrix-0.7-metadata.parquet` and +`matrix-0.7-date-units.parquet` were captured using the original writer at +b2095159664b34ce426685c387adf0bccdb33e01. The opt-in `ParquetLegacyCaptureTest` compiles that +writer and its value helper together in an isolated loader. It records the mixed Date/Timestamp +1000-fold unit corruption. The original writer cannot produce a valid byte[] file here: it fails +with InvalidSchemaException. `legacy-binary-metadata.parquet` is explicitly synthetic old `[B` +metadata with a textual payload; it tests rejection, not recovery of original bytes. + +`hub-urls.tsv` comes from offline parsing and URL generation by huggingface_hub **2.2.0**: + +```bash +python3 -m venv /tmp/matrix-hf-fixtures +/tmp/matrix-hf-fixtures/bin/python -m pip install huggingface_hub==2.2.0 +/tmp/matrix-hf-fixtures/bin/python matrix-parquet/src/test/python/generate_hub_urls.py +``` + +The reviewed v1.0.0 parser uses resolve_path; 2.2.0 uses parse_hf_uri. The later convert-ref +regex accepts dots/dashes (`[\w.-]+`), and revision separation occurs immediately after the +namespaced repository, so deep-path `@` is literal. Special convert/pr refs remain unescaped; +other revision slashes are encoded. Revision text is unquoted once; paths are literal and +quoted, including existing percent signs. The Java parser follows the tested 2.2.0 behavior. + +The additional three-row Arrow fixtures cover an all-null STRING dictionary and JSON logical +annotations. Their file-level manifest rows override the main 4096-row dataset count. diff --git a/matrix-parquet/src/test/resources/optimized/arrow-all-null-string.parquet b/matrix-parquet/src/test/resources/optimized/arrow-all-null-string.parquet new file mode 100644 index 000000000..58ebf06a7 Binary files /dev/null and b/matrix-parquet/src/test/resources/optimized/arrow-all-null-string.parquet differ diff --git a/matrix-parquet/src/test/resources/optimized/arrow-json.parquet b/matrix-parquet/src/test/resources/optimized/arrow-json.parquet new file mode 100644 index 000000000..c00a86047 Binary files /dev/null and b/matrix-parquet/src/test/resources/optimized/arrow-json.parquet differ diff --git a/matrix-parquet/src/test/resources/optimized/cdc-0-index-0-alternate.parquet b/matrix-parquet/src/test/resources/optimized/cdc-0-index-0-alternate.parquet new file mode 100644 index 000000000..917c07fea Binary files /dev/null and b/matrix-parquet/src/test/resources/optimized/cdc-0-index-0-alternate.parquet differ diff --git a/matrix-parquet/src/test/resources/optimized/cdc-0-index-0.parquet b/matrix-parquet/src/test/resources/optimized/cdc-0-index-0.parquet new file mode 100644 index 000000000..1bfb06e76 Binary files /dev/null and b/matrix-parquet/src/test/resources/optimized/cdc-0-index-0.parquet differ diff --git a/matrix-parquet/src/test/resources/optimized/cdc-0-index-1-alternate.parquet b/matrix-parquet/src/test/resources/optimized/cdc-0-index-1-alternate.parquet new file mode 100644 index 000000000..781b2c48a Binary files /dev/null and b/matrix-parquet/src/test/resources/optimized/cdc-0-index-1-alternate.parquet differ diff --git a/matrix-parquet/src/test/resources/optimized/cdc-0-index-1.parquet b/matrix-parquet/src/test/resources/optimized/cdc-0-index-1.parquet new file mode 100644 index 000000000..90aec545d Binary files /dev/null and b/matrix-parquet/src/test/resources/optimized/cdc-0-index-1.parquet differ diff --git a/matrix-parquet/src/test/resources/optimized/cdc-1-index-0-alternate.parquet b/matrix-parquet/src/test/resources/optimized/cdc-1-index-0-alternate.parquet new file mode 100644 index 000000000..8d6380d0d Binary files /dev/null and b/matrix-parquet/src/test/resources/optimized/cdc-1-index-0-alternate.parquet differ diff --git a/matrix-parquet/src/test/resources/optimized/cdc-1-index-0.parquet b/matrix-parquet/src/test/resources/optimized/cdc-1-index-0.parquet new file mode 100644 index 000000000..535f52a00 Binary files /dev/null and b/matrix-parquet/src/test/resources/optimized/cdc-1-index-0.parquet differ diff --git a/matrix-parquet/src/test/resources/optimized/cdc-1-index-1-alternate.parquet b/matrix-parquet/src/test/resources/optimized/cdc-1-index-1-alternate.parquet new file mode 100644 index 000000000..efbc14b45 Binary files /dev/null and b/matrix-parquet/src/test/resources/optimized/cdc-1-index-1-alternate.parquet differ diff --git a/matrix-parquet/src/test/resources/optimized/cdc-1-index-1.parquet b/matrix-parquet/src/test/resources/optimized/cdc-1-index-1.parquet new file mode 100644 index 000000000..8ef275c63 Binary files /dev/null and b/matrix-parquet/src/test/resources/optimized/cdc-1-index-1.parquet differ diff --git a/matrix-parquet/src/test/resources/optimized/hub-urls.tsv b/matrix-parquet/src/test/resources/optimized/hub-urls.tsv new file mode 100644 index 000000000..a53b9d63c --- /dev/null +++ b/matrix-parquet/src/test/resources/optimized/hub-urls.tsv @@ -0,0 +1,13 @@ +# huggingface_hub=2.2.0 +hf://datasets/owner/repo/train.parquet https://huggingface.co/datasets/owner/repo/resolve/main/train.parquet +hf://datasets/owner/repo@v1/train.parquet https://huggingface.co/datasets/owner/repo/resolve/v1/train.parquet +hf://datasets/owner/repo@feature%2Fbranch/train.parquet https://huggingface.co/datasets/owner/repo/resolve/feature%2Fbranch/train.parquet +hf://datasets/owner/repo@refs/convert/parquet/train.parquet https://huggingface.co/datasets/owner/repo/resolve/refs/convert/parquet/train.parquet +hf://datasets/owner/repo@refs%2Fconvert%2Fparquet/train.parquet https://huggingface.co/datasets/owner/repo/resolve/refs/convert/parquet/train.parquet +hf://datasets/owner/repo@refs/pr/10/train.parquet https://huggingface.co/datasets/owner/repo/resolve/refs/pr/10/train.parquet +hf://datasets/owner/repo@refs/convert/parquet-v2/train.parquet https://huggingface.co/datasets/owner/repo/resolve/refs/convert/parquet-v2/train.parquet +hf://datasets/owner/repo/sub/file@1.parquet https://huggingface.co/datasets/owner/repo/resolve/main/sub/file%401.parquet +hf://datasets/owner/repo@v%2B1/café x.parquet https://huggingface.co/datasets/owner/repo/resolve/v%2B1/caf%C3%A9%20x.parquet +hf://datasets/owner/repo/file%20x.parquet https://huggingface.co/datasets/owner/repo/resolve/main/file%2520x.parquet +hf://datasets/owner/repo@v*/file*.parquet https://huggingface.co/datasets/owner/repo/resolve/v%2A/file%2A.parquet +hf://datasets/owner/repo@feature%252Fbranch/a%2Fb.parquet https://huggingface.co/datasets/owner/repo/resolve/feature%252Fbranch/a%252Fb.parquet diff --git a/matrix-parquet/src/test/resources/optimized/legacy-binary-metadata.parquet b/matrix-parquet/src/test/resources/optimized/legacy-binary-metadata.parquet new file mode 100644 index 000000000..d92a0d34d Binary files /dev/null and b/matrix-parquet/src/test/resources/optimized/legacy-binary-metadata.parquet differ diff --git a/matrix-parquet/src/test/resources/optimized/manifest.json b/matrix-parquet/src/test/resources/optimized/manifest.json new file mode 100644 index 000000000..08a8228b5 --- /dev/null +++ b/matrix-parquet/src/test/resources/optimized/manifest.json @@ -0,0 +1,196 @@ +{ + "pyarrow": "21.0.0", + "rows": 4096, + "expected": { + "first_id": 0, + "last_id": 4095, + "price_123": "1.23", + "local": "2024-03-31T01:30:00", + "payload_0_hex": "0000ff", + "tags_1": [ + "\u03b1", + null, + "\u03b2" + ] + }, + "files": { + "cdc-0-index-0.parquet": { + "options": { + "compression": "snappy", + "use_dictionary": false, + "data_page_version": "1.0", + "write_page_index": false, + "row_group_size": 2048, + "data_page_size": 4096, + "write_batch_size": 128, + "dictionary_pagesize_limit": 1024, + "use_content_defined_chunking": false + }, + "sha256": "12a973a08998d92a31ea1de7136eeb8d3003a8fe7297a2705f1295231ee4fdc4", + "schema": "id: int64\ntext: string\npayload: binary\nprice: decimal128(12, 2)\nlocal: timestamp[us]\ndate: date32[day]\nnulls: string\nmedia: struct\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" + }, + "cdc-0-index-0-alternate.parquet": { + "options": { + "compression": "zstd", + "use_dictionary": [ + "text", + "tags.list.element" + ], + "data_page_version": "2.0", + "write_page_index": false, + "row_group_size": 2048, + "data_page_size": 4096, + "write_batch_size": 128, + "dictionary_pagesize_limit": 1024, + "use_content_defined_chunking": false + }, + "sha256": "1a956d7b7478b23e2178aa15d74222d85691f67785e6659cc559b3d91738292a", + "schema": "id: int64\ntext: string\npayload: binary\nprice: decimal128(12, 2)\nlocal: timestamp[us]\ndate: date32[day]\nnulls: string\nmedia: struct\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" + }, + "cdc-0-index-1.parquet": { + "options": { + "compression": "snappy", + "use_dictionary": [ + "text", + "tags.list.element" + ], + "data_page_version": "2.0", + "write_page_index": true, + "row_group_size": 2048, + "data_page_size": 4096, + "write_batch_size": 128, + "dictionary_pagesize_limit": 1024, + "use_content_defined_chunking": false + }, + "sha256": "de20b7e5087bfa0d91c8d44618415e1f0b9612c09b75d6872cd07d215eb70d8d", + "schema": "id: int64\ntext: string\npayload: binary\nprice: decimal128(12, 2)\nlocal: timestamp[us]\ndate: date32[day]\nnulls: string\nmedia: struct\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" + }, + "cdc-0-index-1-alternate.parquet": { + "options": { + "compression": "zstd", + "use_dictionary": false, + "data_page_version": "1.0", + "write_page_index": true, + "row_group_size": 2048, + "data_page_size": 4096, + "write_batch_size": 128, + "dictionary_pagesize_limit": 1024, + "use_content_defined_chunking": false + }, + "sha256": "c8cf3a35454a8525e2ef6b9b707fe9e2e8eeab744f5d236ccc5f039a21e6d421", + "schema": "id: int64\ntext: string\npayload: binary\nprice: decimal128(12, 2)\nlocal: timestamp[us]\ndate: date32[day]\nnulls: string\nmedia: struct\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" + }, + "cdc-1-index-0.parquet": { + "options": { + "compression": "zstd", + "use_dictionary": false, + "data_page_version": "2.0", + "write_page_index": false, + "row_group_size": 2048, + "data_page_size": 4096, + "write_batch_size": 128, + "dictionary_pagesize_limit": 1024, + "use_content_defined_chunking": { + "min_chunk_size": 4096, + "max_chunk_size": 16384, + "norm_level": 0 + } + }, + "sha256": "0bb3c02391af5fd082b060d836f00b00ed67b17d238b9715f730e5a45878f79e", + "schema": "id: int64\ntext: string\npayload: binary\nprice: decimal128(12, 2)\nlocal: timestamp[us]\ndate: date32[day]\nnulls: string\nmedia: struct\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" + }, + "cdc-1-index-0-alternate.parquet": { + "options": { + "compression": "snappy", + "use_dictionary": [ + "text", + "tags.list.element" + ], + "data_page_version": "1.0", + "write_page_index": false, + "row_group_size": 2048, + "data_page_size": 4096, + "write_batch_size": 128, + "dictionary_pagesize_limit": 1024, + "use_content_defined_chunking": { + "min_chunk_size": 4096, + "max_chunk_size": 16384, + "norm_level": 0 + } + }, + "sha256": "2c52062818bcf4a8a5ddf329c6401fcaaa2214b4bc5eb7464c78faebb3ee9ba2", + "schema": "id: int64\ntext: string\npayload: binary\nprice: decimal128(12, 2)\nlocal: timestamp[us]\ndate: date32[day]\nnulls: string\nmedia: struct\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" + }, + "cdc-1-index-1.parquet": { + "options": { + "compression": "zstd", + "use_dictionary": [ + "text", + "tags.list.element" + ], + "data_page_version": "1.0", + "write_page_index": true, + "row_group_size": 2048, + "data_page_size": 4096, + "write_batch_size": 128, + "dictionary_pagesize_limit": 1024, + "use_content_defined_chunking": { + "min_chunk_size": 4096, + "max_chunk_size": 16384, + "norm_level": 0 + } + }, + "sha256": "7b08f89d286d43928e4445bb0aa114a2211d1963c7670db5f9a68fa72f7e7bb4", + "schema": "id: int64\ntext: string\npayload: binary\nprice: decimal128(12, 2)\nlocal: timestamp[us]\ndate: date32[day]\nnulls: string\nmedia: struct\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" + }, + "cdc-1-index-1-alternate.parquet": { + "options": { + "compression": "snappy", + "use_dictionary": false, + "data_page_version": "2.0", + "write_page_index": true, + "row_group_size": 2048, + "data_page_size": 4096, + "write_batch_size": 128, + "dictionary_pagesize_limit": 1024, + "use_content_defined_chunking": { + "min_chunk_size": 4096, + "max_chunk_size": 16384, + "norm_level": 0 + } + }, + "sha256": "9061dc6cac1f451ec43aed6ed0fb7392b2d5669b9a5f27ac3aa22a9a187092e3", + "schema": "id: int64\ntext: string\npayload: binary\nprice: decimal128(12, 2)\nlocal: timestamp[us]\ndate: date32[day]\nnulls: string\nmedia: struct\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" + }, + "arrow-all-null-string.parquet": { + "sha256": "12949788c7fc627d17a79937f560e82b4b4b6d7977b220c32bc74e1ceb668e71", + "rows": 3, + "schema": "s: string" + }, + "arrow-json.parquet": { + "sha256": "b017865f5ec5157bae2147e206f0d304d8c24f031968545df3f7e76208397667", + "rows": 3, + "schema": "json: extension" + }, + "legacy-binary-metadata.parquet": { + "provenance": "synthetic legacy [B metadata, not claimed as original recoverable bytes", + "sha256": "f46e7f0cc774068aba6cc44575209f92d8efb01ef2c008a403777c532943e8d1", + "expected": "unrecoverable-legacy-binary error" + }, + "matrix-0.7-strings.parquet": { + "provenance": "original writer at b2095159664b34ce426685c387adf0bccdb33e01", + "sha256": "249bb46f28df655b1634a83a92ac3a2942812906136d41677db7291f867f2f75", + "schema": "text: binary\nnested: map\n child 0, nested: struct not null\n child 0, key: binary not null\n child 1, value: binary\n-- schema metadata --\nmatrix.columnTypes: 'java.lang.String,java.util.Map'\nwriter.model.name: 'example'" + }, + "matrix-0.7-date-units.parquet": { + "provenance": "original writer at b2095159664b34ce426685c387adf0bccdb33e01", + "sha256": "7cd86c9165de30d5d5e27f8476db3e37a40eef590c4a2675f173bf2bcbd05af5", + "schema": "value: timestamp[ms, tz=UTC]\n-- schema metadata --\nmatrix.columnTypes: 'java.util.Date'\nwriter.model.name: 'example'" + }, + "matrix-0.7-metadata.parquet": { + "provenance": "original writer at b2095159664b34ce426685c387adf0bccdb33e01", + "sha256": "09dbb3d63b05d58b494b7c3cf92dfb3233cb77daf7ff49a174ff3ffe2230f59d", + "schema": "amount: decimal128(3, 2)\nlocal: timestamp[us, tz=UTC]\n-- schema metadata --\nmatrix.columnTypes: 'java.math.BigDecimal,java.time.LocalDateTime'\nwriter.model.name: 'example'" + } + } +} diff --git a/matrix-parquet/src/test/resources/optimized/matrix-0.7-date-units.parquet b/matrix-parquet/src/test/resources/optimized/matrix-0.7-date-units.parquet new file mode 100644 index 000000000..7dc81b31d Binary files /dev/null and b/matrix-parquet/src/test/resources/optimized/matrix-0.7-date-units.parquet differ diff --git a/matrix-parquet/src/test/resources/optimized/matrix-0.7-metadata.parquet b/matrix-parquet/src/test/resources/optimized/matrix-0.7-metadata.parquet new file mode 100644 index 000000000..6edf4b888 Binary files /dev/null and b/matrix-parquet/src/test/resources/optimized/matrix-0.7-metadata.parquet differ diff --git a/matrix-parquet/src/test/resources/optimized/matrix-0.7-strings.parquet b/matrix-parquet/src/test/resources/optimized/matrix-0.7-strings.parquet new file mode 100644 index 000000000..dc9c2ee30 Binary files /dev/null and b/matrix-parquet/src/test/resources/optimized/matrix-0.7-strings.parquet differ