From eee2b16e79c92af643b1e5dfa530be85188edf25 Mon Sep 17 00:00:00 2001 From: Per Nyfelt Date: Sat, 17 Oct 2026 20:00:41 +0200 Subject: [PATCH 1/5] Add optimized Parquet support and selective dataset reads --- .github/workflows/release-all-tools.yml | 24 + docs/cookbook/matrix-parquet.md | 26 + docs/tutorial/11-matrix-parquet.md | 26 + .../benchmarks/optimized-parquet.md | 145 ++++ matrix-parquet/build.gradle | 47 +- matrix-parquet/readme.md | 500 +++++------ matrix-parquet/release.md | 51 ++ matrix-parquet/req/optimized-parquet-plan.md | 782 ++++++++++++++++++ .../consumerTest/java/ParquetConsumer.java | 20 + .../matrix/parquet/ColumnMaterializer.groovy | 175 ++++ .../matrix/parquet/ColumnTypePolicy.groovy | 105 +++ .../matrix/parquet/DatasetBatches.groovy | 176 ++++ .../matrix/parquet/DecimalSchemaPolicy.groovy | 37 + .../matrix/parquet/HttpRangeInputFile.groovy | 141 ++++ .../matrix/parquet/HttpTransport.groovy | 60 ++ .../se/alipsa/matrix/parquet/HubUrl.groovy | 61 ++ .../matrix/parquet/InMemoryInputFile.groovy | 34 + .../matrix/parquet/MatrixBatchIterator.groovy | 9 + .../matrix/parquet/MatrixParquetReader.groovy | 435 ++++++---- .../matrix/parquet/MatrixParquetWriter.groovy | 257 +++--- .../matrix/parquet/ParquetBatches.groovy | 210 +++++ .../parquet/ParquetFormatProvider.groovy | 40 +- .../matrix/parquet/ParquetPredicate.groovy | 210 +++++ .../matrix/parquet/ParquetReadOptions.groovy | 135 ++- .../matrix/parquet/ParquetValueWriter.groovy | 25 +- .../matrix/parquet/ParquetWriteOptions.groovy | 84 +- .../alipsa/matrix/parquet/Predicates.groovy | 276 +++++++ .../src/test/groovy/HubUrlTest.groovy | 20 + .../src/test/groovy/MatrixParquetTest.groovy | 19 +- .../test/groovy/OptimizedParquetTest.groovy | 129 +++ .../test/groovy/ParquetBaselineTest.groovy | 27 + .../test/groovy/ParquetCodecProbeTest.groovy | 25 + .../groovy/ParquetDatasetAndHttpTest.groovy | 105 +++ .../src/test/groovy/ParquetInteropTest.groovy | 40 + .../groovy/ParquetLegacyCaptureTest.groovy | 57 ++ .../matrix/parquet/CountingInput.groovy | 32 + .../matrix/parquet/HttpTransportTest.groovy | 97 +++ .../matrix/parquet/LiveHeapSampler.groovy | 43 + .../ParquetDatasetBenchmarkTest.groovy | 38 + .../parquet/ParquetLayoutBenchmarkTest.groovy | 35 + .../matrix/parquet/ParquetPruningTest.groovy | 128 +++ .../parquet/ParquetReadBenchmarkTest.groovy | 74 ++ .../parquet/ParquetTypePolicyTest.groovy | 247 ++++++ .../src/test/python/generate_benchmarks.py | 19 + .../src/test/python/generate_hub_urls.py | 25 + .../src/test/python/generate_optimized.py | 63 ++ .../src/test/python/verify_matrix_output.py | 21 + .../src/test/resources/optimized/.gitignore | 2 + .../src/test/resources/optimized/README.md | 42 + .../optimized/cdc-0-index-0-alternate.parquet | Bin 0 -> 51586 bytes .../resources/optimized/cdc-0-index-0.parquet | Bin 0 -> 116005 bytes .../optimized/cdc-0-index-1-alternate.parquet | Bin 0 -> 50925 bytes .../resources/optimized/cdc-0-index-1.parquet | Bin 0 -> 117517 bytes .../optimized/cdc-1-index-0-alternate.parquet | Bin 0 -> 126744 bytes .../resources/optimized/cdc-1-index-0.parquet | Bin 0 -> 70910 bytes .../optimized/cdc-1-index-1-alternate.parquet | Bin 0 -> 135356 bytes .../resources/optimized/cdc-1-index-1.parquet | Bin 0 -> 65539 bytes .../src/test/resources/optimized/hub-urls.tsv | 13 + .../optimized/legacy-binary-metadata.parquet | Bin 0 -> 597 bytes .../test/resources/optimized/manifest.json | 186 +++++ .../optimized/matrix-0.7-date-units.parquet | Bin 0 -> 404 bytes .../optimized/matrix-0.7-metadata.parquet | Bin 0 -> 592 bytes .../optimized/matrix-0.7-strings.parquet | Bin 0 -> 844 bytes 63 files changed, 4914 insertions(+), 664 deletions(-) create mode 100644 matrix-parquet/benchmarks/optimized-parquet.md create mode 100644 matrix-parquet/req/optimized-parquet-plan.md create mode 100644 matrix-parquet/src/consumerTest/java/ParquetConsumer.java create mode 100644 matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnMaterializer.groovy create mode 100644 matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnTypePolicy.groovy create mode 100644 matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DatasetBatches.groovy create mode 100644 matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DecimalSchemaPolicy.groovy create mode 100644 matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpRangeInputFile.groovy create mode 100644 matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpTransport.groovy create mode 100644 matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HubUrl.groovy create mode 100644 matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/InMemoryInputFile.groovy create mode 100644 matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixBatchIterator.groovy create mode 100644 matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetBatches.groovy create mode 100644 matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetPredicate.groovy create mode 100644 matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/Predicates.groovy create mode 100644 matrix-parquet/src/test/groovy/HubUrlTest.groovy create mode 100644 matrix-parquet/src/test/groovy/OptimizedParquetTest.groovy create mode 100644 matrix-parquet/src/test/groovy/ParquetBaselineTest.groovy create mode 100644 matrix-parquet/src/test/groovy/ParquetCodecProbeTest.groovy create mode 100644 matrix-parquet/src/test/groovy/ParquetDatasetAndHttpTest.groovy create mode 100644 matrix-parquet/src/test/groovy/ParquetInteropTest.groovy create mode 100644 matrix-parquet/src/test/groovy/ParquetLegacyCaptureTest.groovy create mode 100644 matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/CountingInput.groovy create mode 100644 matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/HttpTransportTest.groovy create mode 100644 matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/LiveHeapSampler.groovy create mode 100644 matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetDatasetBenchmarkTest.groovy create mode 100644 matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetLayoutBenchmarkTest.groovy create mode 100644 matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetPruningTest.groovy create mode 100644 matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReadBenchmarkTest.groovy create mode 100644 matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetTypePolicyTest.groovy create mode 100644 matrix-parquet/src/test/python/generate_benchmarks.py create mode 100644 matrix-parquet/src/test/python/generate_hub_urls.py create mode 100644 matrix-parquet/src/test/python/generate_optimized.py create mode 100644 matrix-parquet/src/test/python/verify_matrix_output.py create mode 100644 matrix-parquet/src/test/resources/optimized/.gitignore create mode 100644 matrix-parquet/src/test/resources/optimized/README.md create mode 100644 matrix-parquet/src/test/resources/optimized/cdc-0-index-0-alternate.parquet create mode 100644 matrix-parquet/src/test/resources/optimized/cdc-0-index-0.parquet create mode 100644 matrix-parquet/src/test/resources/optimized/cdc-0-index-1-alternate.parquet create mode 100644 matrix-parquet/src/test/resources/optimized/cdc-0-index-1.parquet create mode 100644 matrix-parquet/src/test/resources/optimized/cdc-1-index-0-alternate.parquet create mode 100644 matrix-parquet/src/test/resources/optimized/cdc-1-index-0.parquet create mode 100644 matrix-parquet/src/test/resources/optimized/cdc-1-index-1-alternate.parquet create mode 100644 matrix-parquet/src/test/resources/optimized/cdc-1-index-1.parquet create mode 100644 matrix-parquet/src/test/resources/optimized/hub-urls.tsv create mode 100644 matrix-parquet/src/test/resources/optimized/legacy-binary-metadata.parquet create mode 100644 matrix-parquet/src/test/resources/optimized/manifest.json create mode 100644 matrix-parquet/src/test/resources/optimized/matrix-0.7-date-units.parquet create mode 100644 matrix-parquet/src/test/resources/optimized/matrix-0.7-metadata.parquet create mode 100644 matrix-parquet/src/test/resources/optimized/matrix-0.7-strings.parquet diff --git a/.github/workflows/release-all-tools.yml b/.github/workflows/release-all-tools.yml index 1e9c70f54..2825b0a90 100644 --- a/.github/workflows/release-all-tools.yml +++ b/.github/workflows/release-all-tools.yml @@ -3,6 +3,7 @@ name: Combined release tools on: pull_request: paths: + - 'matrix-parquet/**' - 'releaseAll.sh' - 'scripts/release-all/**' - 'matrix-bom/verifyBomApi.sh' @@ -73,3 +74,26 @@ jobs: run: | BOM_VERIFY_REPO="$GITHUB_WORKSPACE/build/releaseAll/bom-verify" bash matrix-bom/verifyBomApi.sh groovy scripts/release-all/test-manifest-verifier.groovy . build/releaseAll/bom-verify + + parquet-interop: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - uses: gradle/actions/setup-gradle@v4 + - run: python -m pip install pyarrow==21.0.0 huggingface_hub==2.2.0 + - run: python matrix-parquet/src/test/python/generate_optimized.py + - run: python matrix-parquet/src/test/python/generate_hub_urls.py + - run: git diff --exit-code -- matrix-parquet/src/test/resources/optimized + - run: ./gradlew :matrix-parquet:codenarcMain :matrix-parquet:codenarcTest :matrix-parquet:spotlessCheck :matrix-parquet:test :matrix-parquet:compileConsumerApi -PrunSlowTests=true + env: + PARQUET_PYTHON: python + - run: python matrix-parquet/src/test/python/verify_matrix_output.py + - run: ./gradlew :matrix-parquet:test --tests ParquetInteropTest --rerun-tasks diff --git a/docs/cookbook/matrix-parquet.md b/docs/cookbook/matrix-parquet.md index a7c6ba5b3..41ffdf6b8 100644 --- a/docs/cookbook/matrix-parquet.md +++ b/docs/cookbook/matrix-parquet.md @@ -158,3 +158,29 @@ For files with no Matrix metadata at all, Parquet `DECIMAL` columns are inferred --- [Back to index](cookbook.md) + + +## Optimized Parquet, projection and bounded reads (0.8.0) + +```groovy +import se.alipsa.matrix.parquet.MatrixParquetReader +import se.alipsa.matrix.parquet.Predicates + +def subset = MatrixParquetReader.builder().columns(['id', 'text']) + .filter(Predicates.ge('id', 1000L)).limit(100) + .read(new File('downloaded-optimized.parquet')) + +MatrixParquetReader.builder().maxBatchRows(500).batches(new File('downloaded-optimized.parquet')) + .withCloseable { batches -> + while (batches.hasNext()) { + def batch = batches.next() + // Process this Matrix before requesting another batch. + } + } +``` + +CDC pages decode as standard Parquet; page indexes help eligible predicates skip pages. These APIs +include exact logical-schema predicate conversion and retain selected types for empty results. +See the [complete module guide](../../matrix-parquet/readme.md) for all parameter defaults, +HTTP/Hub access, sharded datasets, writer tuning, fixture reproduction and the external CDC rewrite. +Read the [0.8.0 migration table](../../matrix-parquet/release.md) before upgrading existing calls. diff --git a/docs/tutorial/11-matrix-parquet.md b/docs/tutorial/11-matrix-parquet.md index a11261085..2850a1e6f 100644 --- a/docs/tutorial/11-matrix-parquet.md +++ b/docs/tutorial/11-matrix-parquet.md @@ -261,3 +261,29 @@ The module uses Apache Parquet and Hadoop libraries directly: In the next section, we'll explore the matrix-bigquery module, which provides functionality for interacting with Google BigQuery. Go to [previous section](10-matrix-bom.md) | Go to [next section](12-matrix-bigquery.md) | Back to [outline](outline.md) + + +## Optimized Parquet, projection and bounded reads (0.8.0) + +```groovy +import se.alipsa.matrix.parquet.MatrixParquetReader +import se.alipsa.matrix.parquet.Predicates + +def subset = MatrixParquetReader.builder().columns(['id', 'text']) + .filter(Predicates.ge('id', 1000L)).limit(100) + .read(new File('downloaded-optimized.parquet')) + +MatrixParquetReader.builder().maxBatchRows(500).batches(new File('downloaded-optimized.parquet')) + .withCloseable { batches -> + while (batches.hasNext()) { + def batch = batches.next() + // Process this Matrix before requesting another batch. + } + } +``` + +CDC pages decode as standard Parquet; page indexes help eligible predicates skip pages. These APIs +include exact logical-schema predicate conversion and retain selected types for empty results. +See the [complete module guide](../../matrix-parquet/readme.md) for all parameter defaults, +HTTP/Hub access, sharded datasets, writer tuning, fixture reproduction and the external CDC rewrite. +Read the [0.8.0 migration table](../../matrix-parquet/release.md) before upgrading existing calls. diff --git a/matrix-parquet/benchmarks/optimized-parquet.md b/matrix-parquet/benchmarks/optimized-parquet.md new file mode 100644 index 000000000..f7591f476 --- /dev/null +++ b/matrix-parquet/benchmarks/optimized-parquet.md @@ -0,0 +1,145 @@ +# Optimized Parquet measurements + +Measured 2026-10-17 on macOS arm64 (Apple M5 Max), G1 GC, OpenJDK 21.0.12.1, 4 GiB test heap, parquet-java 1.18.1, +SNAPPY, default 128 MiB row-group target and 1 MiB pages. Each workload has 1,000,000 rows, +an Integer ID and six String columns. Wide text repeats 1,000 labels per column; categorical +repeats 12. Mixed adds LocalDate and Map(label, exact BigDecimal) columns. Generated inputs +live in build/, outside git. The compared Group loop uses shared corrected value conversion. + +| Workload | File bytes | Group median ms | Columnar median ms | Lower median time | Group retained bytes | Columnar retained bytes | +|---|---:|---:|---:|---:|---:|---:| +| wide | 4,898,180 | 7,990 | 767 | 90.4% | 381,674,424 | 45,751,352 | +| categorical | 4,173,171 | 7,940 | 775 | 90.2% | 342,270,976 | 45,364,064 | +| mixed/nested | 7,934,477 | 18,569 | 2,462 | 86.7% | 790,769,704 | 406,370,144 | + +Medians exclude run 0; three warmed runs are included. Group warmed elapsed ranges were +7,368–10,150 / 6,845–11,093 / 18,334–21,881 ms. Columnar ranges were 765–782 / 763–783 / +2,332–2,486 ms. One codec-spike build overlapped the wide Group run and contributes to its +variance; these are local development measurements, not universal speed guarantees. The +original pre-change reader baseline was separately captured before production edits at +5,633 / 5,413 / 5,433 / 5,424 ms on the same million-row wide shape (file 4,898,111 bytes). + +Current-thread allocated bytes were approximately 13.83 GB versus 1.32 GB (wide), 13.78 GB +versus 1.31 GB (categorical), and 38.53 GB versus 3.52 GB (mixed). Retained deltas are sampled +after explicit full GC with the result live. They measure retained result heap, not peak RSS +or a rigorously sampled peak-live-heap metric. In particular the categorical retained result +uses 86.7% less heap; do not relabel that measurement as peak heap. No OS cache purge was +performed. Run 0 represents first application access, not a guaranteed cold filesystem cache. +Timing thresholds are absent from correctness CI. + +## Confirmation with GC live-heap sampling + +A second million-row run added GC notifications. The sampler records the maximum sum of heap +pool usage **after GC** during each read, including the final full GC with the result live. +This is sampled peak live heap, including benchmark inputs/buffers, not instantaneous heap +occupancy between GCs or process RSS. Medians below exclude run 0 (three warmed runs). + +| Workload | Group ms | Columnar ms | Group sampled live peak bytes | Columnar sampled live peak bytes | +|---|---:|---:|---:|---:| +| wide | 7,280 | 764 | 1,009,640,184 | 584,545,248 | +| categorical | 7,158 | 797 | 963,066,680 | 541,390,488 | +| mixed/nested | 18,936 | 2,238 | 1,998,185,928 | 1,343,803,744 | + +Wide elapsed time is 89.5% lower; categorical sampled live heap is 43.8% lower; mixed elapsed +time is 88.2% lower. Thus the initial 20% elapsed / 25% live-heap / at-most-5% regression targets +are met on these workloads with this stated sampling method. Warmed elapsed ranges were +7,230–7,295 / 6,922–7,263 / 18,605–19,304 ms for Group and 752–786 / 776–800 / 2,213–2,306 ms +for columnar. Allocations were approximately 14.20/1.32, 14.15/1.31 and 40.16/3.40 GB respectively. +These local measurements remain evidence, not performance guarantees or CI timing assertions. + +## Reproduction + +Create an init file outside the repository: + +```groovy +// /tmp/parquet-benchmark.init.gradle +gradle.projectsEvaluated { + gradle.rootProject.project(':matrix-parquet').tasks.named('test') { + systemProperty 'parquet.baseline', 'true' + maxHeapSize = '4g' + } +} +``` + +```bash +./gradlew :matrix-parquet:test --tests '*ParquetReadBenchmarkTest' \ + -I /tmp/parquet-benchmark.init.gradle --no-configuration-cache --console=plain +``` + +The BENCH records include workload, implementation, run, rows, file bytes, elapsed milliseconds, +thread allocation and retained heap. Avoid concurrent builds for isolated timing measurements. + +## Correctness and I/O evidence + +`ParquetPruningTest` inspects actual offset/column indexes and multiple pages, compares indexed +and nonindexed byte counts with row-group/statistics/dictionary/bloom pruning disabled, and +checks dictionary String identity. `ParquetDatasetAndHttpTest` counts transferred range bytes +for narrow projection and selective filtering, checks fallback and ETag changes, and compares +ordered sequential/parallel shard results. These assertions use bytes/results, not wall clocks. + +The no-Hadoop spike (`./gradlew :matrix-parquet:codecWithoutHadoop`) found that SNAPPY, ZSTD, +GZIP and UNCOMPRESSED writer initialization all require +`org/apache/hadoop/io/compress/zlib/ZlibCompressor$CompressionLevel`. Therefore Hadoop remains +on the runtime classpath even though local InputFile/OutputFile use PlainParquetConfiguration. + +The public Java consumer probe (`./gradlew :matrix-parquet:compileConsumerApi`) removes +parquet-java and Hadoop from its compile classpath and successfully compiles projection, +Predicates, limits and writer tuning. This verifies that new public predicates do not require +consumers to add parquet-column explicitly. + +## Bounded dataset parallelism + +Four shards of 250,000 rows each (Integer ID and 1,000 categorical text labels), SNAPPY, +maxBatchRows=65,536, all 1,000,000 rows materialized in deterministic order: + +| Workers | Warm ms (runs 1–3) | Median ms | Median sampled live peak bytes | +|---|---|---:|---:| +| 1 | 370 / 341 / 308 | 341 | 66,905,816 | +| 4 | 247 / 242 / 239 | 242 | 57,235,736 | + +This local workload gains about 29% median throughput time with four workers. Heap samples vary +with collection timing; the measured queue stays bounded to one batch per worker, not a claim +that parallel reads always use less memory. The benchmark verifies first/last IDs and row count. +Run `*ParquetDatasetBenchmarkTest` with the same init setup but system property +`parquet.datasetBenchmark=true` instead of `parquet.baseline`. + +Observed deterministic I/O checks on the controlled fixtures: page-index reads 2,914 bytes +versus 82,700 without page pruning; bloom-filter reads 16,737 versus 40,373 with statistics +unable to exclude the interior key and dictionaries/pages disabled; selective HTTP transferred +88,476 versus a 906,195-byte full object. HTTP data uses generated UUID strings, so exact byte +counts vary; assertions require reduced transfer and exact returned rows, not these constants. + +## Identical Arrow layouts: full scan, projection and predicate + +PyArrow 21.0.0 generated 1,000,000 sorted int64 IDs and six text columns (1,000 repeated labels, +about 140 characters/label), four 250,000-row groups, SNAPPY, 64 KiB target pages, default +dictionary encoding and default CDC sizes. Only CDC/index settings differ. No OS cache purge; +run 0 is warmup, medians include runs 1–3. Heap is the same GC live sample defined above. + +| Layout | Mode | Returned rows | File bytes | Read bytes (footer included) | Median ms | Median live peak bytes | +|---|---|---:|---:|---:|---:|---:| +| ordinary | full | 1,000,000 | 6,066,382 | 6,066,378 | 727 | 110,855,320 | +| ordinary | id projection | 1,000,000 | 6,066,382 | 5,085,954 | 214 | 55,266,648 | +| ordinary | id >= 999900 | 100 | 6,066,382 | 1,279,020 | 99 | 32,019,888 | +| indexed | full | 1,000,000 | 6,069,245 | 6,021,834 | 741 | 117,800,072 | +| indexed | id projection | 1,000,000 | 6,069,245 | 5,082,882 | 226 | 56,010,848 | +| indexed | id >= 999900 | 100 | 6,069,245 | 552,869 | 7 | 27,740,176 | +| CDC + indexed | full | 1,000,000 | 7,250,428 | 6,968,991 | 693 | 111,151,624 | +| CDC + indexed | id projection | 1,000,000 | 7,250,428 | 5,105,311 | 209 | 57,589,672 | +| CDC + indexed | id >= 999900 | 100 | 7,250,428 | 542,383 | 3 | 27,766,232 | + +The selective read benefits from row-group pruning in every layout and page pruning where +indexes exist. These short timings demonstrate this workload, not a guaranteed CDC speedup; +CDC has a larger file here. The independent page-index test isolates page pruning alone. + +Reproduce with pinned PyArrow and the same init script as above, replacing its system property +with `parquet.layoutBenchmark=true`: + +```bash +/tmp/matrix-parquet-fixtures/bin/python matrix-parquet/src/test/python/generate_benchmarks.py +./gradlew :matrix-parquet:test --tests '*ParquetLayoutBenchmarkTest' \ + -I /tmp/parquet-layout.init.gradle --no-configuration-cache --console=plain +``` + +The generator writes only build/layout-benchmarks. The optional test checks returned counts and +last IDs and emits per-run time, exact input bytes and sampled live heap for every combination. diff --git a/matrix-parquet/build.gradle b/matrix-parquet/build.gradle index 177374de0..094566f83 100644 --- a/matrix-parquet/build.gradle +++ b/matrix-parquet/build.gradle @@ -7,7 +7,7 @@ plugins { } group = 'se.alipsa.matrix' -version = '0.7.1-SNAPSHOT' +version = '0.8.0-SNAPSHOT' description = "Groovy library for importing and exporting parquet files to and from a Matrix" JavaCompile javaCompile = compileJava { @@ -124,3 +124,48 @@ nexusReleasePlugin { password = project.ext.properties.sonatypePassword mavenPublication = publishing.publications.maven } + +// Generate the default-size CDC case outside the repository only for the slow lane. +boolean runCdcSlowTests = providers.gradleProperty('runSlowTests').orNull == 'true' +File parquetProjectDirectory = projectDir +tasks.register('generateDefaultCdcFixture') { + onlyIf { runCdcSlowTests } + doLast { + String python = System.getenv('PARQUET_PYTHON') ?: 'python3' + try { + def probe = new ProcessBuilder(python, '-c', 'import pyarrow; assert pyarrow.__version__ == "21.0.0"').inheritIO().start() + if (probe.waitFor() != 0) { + logger.lifecycle('SKIP default CDC fixture: pinned PyArrow 21.0.0 is unavailable') + return + } + } catch (IOException ignored) { + logger.lifecycle('SKIP default CDC fixture: Python is unavailable') + return + } + def generation = new ProcessBuilder(python, 'src/test/python/generate_optimized.py', + 'build/default-cdc', '--default-cdc').directory(parquetProjectDirectory).inheritIO().start() + if (generation.waitFor() != 0) { throw new GradleException('CDC fixture generation failed') } + } +} +tasks.named('test') { dependsOn 'generateDefaultCdcFixture' } + +// Diagnostic spike: report runtime classes still required by upstream codec/writer code. +tasks.register('codecWithoutHadoop', Test) { + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath.filter { !it.name.startsWith('hadoop-') } + useJUnitPlatform() + include '**/ParquetCodecProbeTest.class' + systemProperty 'parquet.codecSpike', 'true' + testLogging.showStandardStreams = true +} + +tasks.register('compileConsumerApi', JavaCompile) { + dependsOn classes + source = fileTree('src/consumerTest/java') { include '**/*.java' } + classpath = sourceSets.test.compileClasspath.filter { + !it.name.startsWith('parquet-') && !it.name.startsWith('hadoop-') + } + sourceSets.main.output + destinationDirectory = layout.buildDirectory.dir('consumer-api') + options.release = 21 +} +tasks.named('check') { dependsOn 'compileConsumerApi' } diff --git a/matrix-parquet/readme.md b/matrix-parquet/readme.md index 9ddfcd525..6986680f4 100644 --- a/matrix-parquet/readme.md +++ b/matrix-parquet/readme.md @@ -1,340 +1,230 @@ -# Matrix-Parquet +# Matrix Parquet -This module enables import and export of [Apache Parquet](https://parquet.apache.org/) files to and from Matrix objects using `MatrixParquetWriter` and `MatrixParquetReader`. +Read and write Apache Parquet from Groovy/Java Matrix applications on JDK 21. This development +version targets **0.8.0** and uses parquet-java 1.18.1. Include `matrix-core` and Groovy alongside +`matrix-parquet` in your application. To test this checkout, run `./gradlew :matrix-parquet:test`. -## At a Glance +## Optimized Parquet and selective reads -| Goal | Entry point | -|------|-------------| -| Read a file | `MatrixParquetReader.builder().read(file)` | -| Write a file | `MatrixParquetWriter.builder(matrix).write(file)` | -| Read via SPI | `Matrix.read(file)` | -| Write via SPI | `matrix.write(file)` | -| Discover read options | `Matrix.listReadOptions('parquet')` / `ParquetReadOptions.describe()` | -| Discover write options | `Matrix.listWriteOptions('parquet')` / `ParquetWriteOptions.describe()` | +Hugging Face ordinary and optimized downloads are standard Parquet. Content-defined chunking +(CDC) changes writer page boundaries for deduplication; it requires no special reader. Page +indexes are independent and let compatible predicates skip pages. The checked-in PyArrow 21 +fixtures exercise all four CDC/index combinations with SNAPPY/ZSTD and V1/V2 pages. -## API Surface - -**Direct API (recommended)** -- `MatrixParquetReader.builder()` — fluent reader with `.matrixName()`, `.zoneId()`, `.read(File/Path/URL/InputStream/byte[])` -- `MatrixParquetWriter.builder(matrix)` — fluent writer with `.precision()`, `.scale()`, `.decimalMeta()`, `.roundingMode()`, `.compressionCodec()`, `.zoneId()`, `.inferPrecisionAndScale()`, `.write(File/Path/OutputStream)`, `.writeBytes()` - -**SPI (generic Matrix API)** -- `Matrix.read(options, file)` — options map with keys `matrixName`, `zoneId` -- `matrix.write(options, file)` — options map with keys `inferPrecisionAndScale`, `precision`, `scale`, `decimalMeta`, `roundingMode`, `compressionCodec`, `zoneId` - -**Typed options classes** -- `ParquetReadOptions` — typed options for reading; use `ParquetReadOptions.describe()` for runtime discovery -- `ParquetWriteOptions` — typed options for writing; use `ParquetWriteOptions.describe()` for runtime discovery - -## Default Behavior - -**Reading** -- Matrix name for file reads: `matrixName` option wins; otherwise the file basename without extension (e.g. `data.parquet` → matrix named `data`) -- Matrix name for stream/byte-array/URL reads: `matrixName` option wins; otherwise the Parquet schema/message name when available -- Timezone: system default; override with `zoneId` option - -**Writing** -- Parquet message type name: `matrix.matrixName` when present, otherwise `MatrixSchema` -- BigDecimal: precision and scale are inferred from the data by default (`inferPrecisionAndScale = true`) -- Explicit DECIMAL scale conversion: `roundingMode = UNNECESSARY` by default, so a discarded non-zero fraction fails instead of being silently rounded -- Compression: `SNAPPY` by default; override with the `compressionCodec` option (e.g. `GZIP`, `ZSTD`, `UNCOMPRESSED`) -- Timezone: system default; override with `zoneId` option -- When the target is an existing directory, the Matrix name is converted to one safe filename inside it. Path separators and unsafe characters are replaced, trailing dots/spaces are removed, blank or dots-only names become `matrix.parquet`, and an existing `.parquet` suffix is preserved. Pass an explicit file (including a non-existent file path) to retain the exact filename. -- Nested Map columns: stored as MAP when value types are homogeneous, as STRUCT when heterogeneous -- Index columns: written to `matrix.indexColumns` metadata as a JSON string array so column names containing commas round-trip; readers also accept the legacy comma-delimited metadata form - -## Installation - -Add the following to your Gradle build script: -```groovy -implementation 'org.apache.groovy:groovy:5.0.6' -implementation 'se.alipsa.matrix:matrix-core:3.9.0' -implementation 'se.alipsa.matrix:matrix-parquet:0.7.0' -``` - -## Basic Usage - -### Using Matrix.read() / matrix.write() - -If `matrix-parquet` is on the classpath, `.parquet` files are also available through the generic Matrix SPI API: - -```groovy -import se.alipsa.matrix.core.Matrix - -Matrix data = Matrix.read(new File('data.parquet')) -Matrix stockholm = Matrix.read([matrixName: 'events', zoneId: 'Europe/Stockholm'], new File('events.parquet')) - -data.write([inferPrecisionAndScale: true], new File('copy.parquet')) -data.write([precision: 38, scale: 18, zoneId: 'Europe/Stockholm'], new File('money.parquet')) -data.write([decimalMeta: [amount: [8, 2]]], new File('money-meta.parquet')) - -println Matrix.listReadOptions('parquet') -println Matrix.listWriteOptions('parquet') -``` - -### Reading a Parquet file ```groovy +import java.time.LocalDate import se.alipsa.matrix.core.Matrix import se.alipsa.matrix.parquet.MatrixParquetReader - -File file = new File("data.parquet") -Matrix matrix = MatrixParquetReader.read(file) - -// Or with a custom matrix name -Matrix matrix = MatrixParquetReader.read(file, "myData") - -// Read from a Path -Matrix matrix = MatrixParquetReader.read(file.toPath()) - -// Read from an InputStream -new FileInputStream(file).withCloseable { is -> - Matrix matrix = MatrixParquetReader.read(is) -} - -// Read from a URL (e.g. file:// or https://) -URL url = file.toURI().toURL() -Matrix matrix = MatrixParquetReader.read(url) - -// Read from a file path string -Matrix matrix = MatrixParquetReader.readFile("/path/to/data.parquet") -``` - -### Writing a Matrix to a Parquet file -```groovy -import se.alipsa.matrix.core.Matrix import se.alipsa.matrix.parquet.MatrixParquetWriter -import se.alipsa.matrix.datasets.Dataset +import se.alipsa.matrix.parquet.Predicates -Matrix data = Dataset.cars().withMatrixName('cars') -File file = new File("cars.parquet") -MatrixParquetWriter.write(data, file) +File file = new File('events.parquet') +Matrix events = Matrix.builder('events').columns( + id: [1L, 2L, 3L], date: [LocalDate.of(2026, 1, 1)] * 3, + text: ['first', 'second', 'third']).build() +MatrixParquetWriter.builder(events).pageSize(65536).write(file) -// Or write to a directory (filename derived from matrix name) -File dir = new File("output/") -MatrixParquetWriter.write(data, dir) // Creates output/cars.parquet +Matrix selected = MatrixParquetReader.builder().columns(['text', 'id']) + .filter(Predicates.ge('id', 2L)).limit(10).read(file) +assert selected.column('id') == [2L, 3L] ``` -### Builder API (recommended) +Projection accepts top-level fields in output order. Filter-only columns are read internally. +`Predicates.eq/ne/lt/le/gt/ge` have Number, String, Boolean, LocalDate, LocalDateTime, Instant, +LocalTime and byte[] overloads. Use `isNull`, `isNotNull`, `and`, `or` and `not` for composition. +Values must match the file's logical schema. Integer overflow, fractional integer values, +lossy decimal rescaling, and timestamp/time precision loss are rejected before reading pages. +Repeated/nested predicates are rejected. Missing indexes fall back to scans; corrupt data fails. + +String ordering uses unsigned UTF-8 bytes, matching Parquet. Unannotated BINARY predicates take +byte[] even when legacy metadata or `binaryAsString` makes output text. Comparisons with null are false; null tests are explicit. `not` is Boolean negation, including +for null rows. NaN comparisons are false except `ne`; numeric infinities retain their ordering. +Safe physical comparisons use row-group, dictionary, bloom and page-index pruning. Binary +DECIMAL comparisons and expressions without safe pushdown use exact residual evaluation. +LocalDateTime predicates on adjusted timestamps use `zoneId`; Instant predicates on naive +local timestamps are rejected. External TIME fields retain the existing `java.sql.Time` +millisecond output contract; filtering evaluates the original time precision. -The builder API provides a fluent, discoverable interface for reading and writing: +## Batches and shards ```groovy -import se.alipsa.matrix.core.Matrix import se.alipsa.matrix.parquet.MatrixParquetReader -import se.alipsa.matrix.parquet.MatrixParquetWriter -// Writing -MatrixParquetWriter.builder(matrix) - .precision(38) - .scale(18) - .zoneId('Europe/Stockholm') - .write(new File('output.parquet')) - -// Writing to byte array -byte[] bytes = MatrixParquetWriter.builder(matrix) - .inferPrecisionAndScale(true) - .writeBytes() - -// Writing to OutputStream -outputStream.withCloseable { os -> - MatrixParquetWriter.builder(matrix).write(os) -} - -// Reading -Matrix data = MatrixParquetReader.builder() - .matrixName('myData') - .zoneId('Europe/Stockholm') - .read(new File('data.parquet')) - -// Reading from byte array, InputStream, URL, or Path also supported -Matrix data = MatrixParquetReader.builder() - .read(parquetBytes) +MatrixParquetReader.builder().limit(1000).maxBatchRows(250) + .batches(new File('events.parquet')).withCloseable { batches -> + while (batches.hasNext()) { + def batch = batches.next() + assert batch.rowCount() <= 250 + } + } + +// Explicit lists retain caller order; directories and explicit globs sort normalized paths. +def dataset = MatrixParquetReader.builder().parallelism(2).sourceFileColumn('_source') + .read([new File('train-00001.parquet'), new File('train-00000.parquet')]) +def fromDirectory = MatrixParquetReader.read(new File('train')) +def fromGlob = MatrixParquetReader.readGlob('train/train-*.parquet') ``` -## Supported Data Types - -| Java/Groovy Type | Parquet Type | Notes | -|--------------------|--------------------------------|---------------------------------------------------| -| Integer | INT32 | | -| Long | INT64 | | -| BigInteger | BINARY (DECIMAL, scale=0) | Precision auto-inferred from data; no `Long` range limit | -| Float | FLOAT | | -| Double | DOUBLE | | -| BigDecimal | FIXED_LEN_BYTE_ARRAY (DECIMAL) | See [BigDecimal Precision](#bigdecimal-precision) | -| Boolean | BOOLEAN | | -| LocalDate | INT32 (DATE) | | -| java.sql.Date | INT32 (DATE) | | -| java.sql.Time | INT32 (TIME_MILLIS) | Millisecond precision; external TIME(MICROS/NANOS) is downgraded to milliseconds | -| LocalDateTime | INT64 (TIMESTAMP_MICROS) | Microsecond precision | -| java.sql.Timestamp | INT64 (TIMESTAMP_MICROS) | Microsecond precision | -| java.util.Date | INT64 | Epoch milliseconds | -| List | LIST | See [Nested Structures](#nested-structures) | -| Map | MAP or STRUCT | See [Nested Structures](#nested-structures) | -| Other | BINARY (STRING) | Stored as toString() | - -## BigDecimal Precision - -BigDecimal values require special handling to preserve precision. There are three approaches: - -### 1. Automatic Precision Inference (Default) -```groovy -// Automatically infers precision and scale from the data -MatrixParquetWriter.write(data, file) -// Or explicitly: -MatrixParquetWriter.write(data, file, true) -``` +A batch normally contains one selected row group. `maxBatchRows` splits large groups; individual +cells and compressed row-group pages still need heap. Close iterators on early exit; exhaustion +and errors close them automatically. Limits count accepted rows across batches/files. Footer +IDs remain valid when filters prune their groups; such groups produce no batch. There is no +ordinal row seeking API. -### 2. Uniform Precision for All BigDecimal Columns -```groovy -// Use precision=38, scale=18 for all BigDecimal columns -MatrixParquetWriter.write(data, file, 38, 18) -``` +Datasets validate every footer before returning rows and require identical logical fields, +order, decimal precision/scale, timestamp units/UTC flags and Matrix type metadata. Duplicate +canonical files and empty selections fail. Directory reads include direct `.parquet` children; +globs are explicit and can select deeper paths. Dataset indices are omitted. A source column +contains canonical absolute paths and must not collide with a physical column. Parallel workers +use independent streams and converters, reuse each parsed footer, prefetch at most one batch per +worker, and preserve file/group order. `rowGroups` is for one file; use +`rowGroupsByFile: [(file.canonicalPath): [1, 0]]` for explicit dataset group selections. -Explicit precision/scale writes reject a value that would lose a non-zero fraction by default. To opt in to a deliberate rounding policy, use the builder or the typed options map: +## HTTP and Hub URIs ```groovy -import java.math.RoundingMode - -// Direct API -MatrixParquetWriter.write(data, file, 8, 2, RoundingMode.HALF_UP) - -// Builder API -MatrixParquetWriter.builder(data) - .precision(8) - .scale(2) - .roundingMode(RoundingMode.HALF_UP) - .write(file) +import se.alipsa.matrix.parquet.MatrixParquetReader +import se.alipsa.matrix.parquet.Predicates -data.write([precision: 8, scale: 2, roundingMode: 'HALF_UP'], file) +def remote = MatrixParquetReader.builder().remoteReadMode('AUTO').columns(['id']) + .filter(Predicates.ge('id', 1000L)).limit(100) + .readUrl('https://huggingface.co/datasets/owner/repo/resolve/main/train.parquet') +def hub = MatrixParquetReader.builder().remoteReadMode('AUTO').limit(20) + .readUrl('hf://datasets/owner/repo@refs/convert/parquet/train/0000.parquet') ``` -`roundingMode` accepts any `RoundingMode` enum name, case-insensitively. Use `UNNECESSARY` (the default) for lossless conversion; a rounded result must still fit the declared precision. +Replace owner/repo/path with a real dataset. DOWNLOAD preserves full-download behavior. AUTO +verifies `206 Content-Range` support then uses ranges, or downloads when the initial probe is +unsupported. RANGE requires range support. Once decoding starts, transport/object-change errors +propagate. Range caching is bounded (individual fetches capped at 64 KiB). ETag/Last-Modified +validators and total length detect changes when available; same-length changes without validators +cannot be detected reliably. Redirects are bounded and HTTPS downgrades rejected. Connection failures before response headers +are retried once; malformed responses and body failures propagate. Streams are spooled to temporary files; byte arrays use a seekable memory input. + +Hub dataset URI parsing is pinned to huggingface_hub v2.2.0. Revisions follow the repository: +`hf://datasets/owner/repo@revision/path`. Encode ordinary revision slashes as `%2F`; special +`refs/convert/` and `refs/pr/N` forms may remain unescaped. `@` inside a file path is literal. +HF URI paths are literal (already-percent-encoded path text is escaped again). `HF_TOKEN` is +optional and read only for explicit hf:// calls; authorization is attached only to HTTPS +`huggingface.co` on its default/443 port and never to CDN redirect origins. This is HTTP range +access, with no Xet download/deduplication client. + +## Read options + +Use fluent builder methods, `MatrixParquetReader.builder(typedOptions)`, or SPI maps: +`Matrix.read([columns: ['id'], limit: 10L], file)`. Unknown options are rejected. +`ParquetReadOptions.describe()` lists these options at runtime. + +| Option | Default | Meaning | +|---|---|---| +| matrixName | null | Explicit name; otherwise filename for local files, schema name for memory/streams/URLs | +| zoneId | system zone (null) | Adjusted timestamp conversion; ZoneId or zone name | +| columns | null/all | Nonempty unique ordered top-level output names | +| filter | null | Immutable Matrix-owned ParquetPredicate | +| binaryAsString | null/AUTO | Legacy Matrix unannotated binary is text; external/new binary is bytes; true reads external unannotated binary as UTF-8 recursively, false reads raw bytes; annotations and explicit new binary metadata take precedence | +| useColumnIndexFilter | true | Permit page pruning; false is useful for result/I/O comparisons | +| limit | null/unlimited | Nonnegative matching-row bound; zero retains schema | +| rowGroups | null/all | Unique, zero-based original footer IDs, caller order | +| rowGroupsByFile | empty/all | Dataset-only canonical file path to ordered group ID list | +| maxBatchRows | null | Positive accepted-row batch cap; default one row group | +| sourceFileColumn | null | Optional dataset path column, added after projection | +| parallelism | 1 | Positive maximum dataset workers across files and row groups | +| remoteReadMode | DOWNLOAD | DOWNLOAD, AUTO or RANGE | +| connectTimeoutMillis | 10000 | Positive HTTP connection timeout | +| readTimeoutMillis | 60000 | Positive HTTP read timeout | +| rangeCacheBytes | 8388608 | Positive per-stream cache ceiling; actual fetch/cache is at most 64 KiB | + +## Writer options and exact types -### 3. Per-Column Precision Specification ```groovy -// Specify precision and scale for specific columns -MatrixParquetWriter.write(data, file, [ - salary: [10, 2], // precision=10, scale=2 - rate: [8, 4] // precision=8, scale=4 -]) -``` - -**Note:** If the specified precision is too small for a value, an `IllegalArgumentException` will be thrown with a descriptive message indicating the required vs. allowed precision. - -## Nested Structures - -The module supports nested Groovy collections: +import se.alipsa.matrix.core.Matrix +import se.alipsa.matrix.parquet.MatrixParquetWriter -```groovy -def data = Matrix.builder('products').data( - id: [1, 2], - tags: [['electronics', 'sale'], ['furniture']], // List - attributes: [[color: 'red', size: 'L'], [color: 'blue']], // Map as STRUCT - reviews: [[[rating: 5, user: 'A']], [[rating: 4, user: 'B']]] // List of Maps -).types([Integer, List, Map, List]).build() - -MatrixParquetWriter.write(data, file) -Matrix restored = MatrixParquetReader.read(file) +def values = Matrix.builder('prices').columns(id: [1L, 2L], price: [1.25G, 2.50G]).build() +MatrixParquetWriter.builder(values).writerVersion('V2').compressionCodec('ZSTD') + .rowGroupSize(16L * 1024 * 1024).pageSize(65536) + .bloomFilterEnabled([id: true]).bloomFilterNDV([id: 1000L]) + .decimalMeta([price: [8, 2] as int[]]).write(new File('prices.parquet')) ``` -### Nested Structure Limitations -- Maps with heterogeneous value types are stored as STRUCT (preserves key names as field names) -- Maps with homogeneous value types are stored as MAP (standard Parquet map) -- Deeply nested structures are supported but may impact performance -- Bean/POJO objects are converted to Maps using reflection - -## Timezone Handling - -By default, `LocalDateTime` values are converted to/from UTC using the system default timezone. To use a specific timezone: - -```groovy -import java.time.ZoneId - -// Writing with a specific timezone -MatrixParquetWriter.write(data, file, ZoneId.of("America/New_York")) - -// Reading with a specific timezone -Matrix matrix = MatrixParquetReader.read(file, ZoneId.of("America/New_York")) - -// Reading with name and timezone -Matrix matrix = MatrixParquetReader.read(file, "myData", ZoneId.of("Europe/London")) +File, Path, byte-array and OutputStream writes share writer tuning. Size settings are targets, +not hard memory limits. V2 changes value encoders as well as page layout; V1 remains the default +for compatibility with older consumers. Bloom NDVs affect memory/false-positive tradeoffs and do +not enable a filter by themselves. Column settings use physical primitive paths (for example +`tags.list.element`); unknown/nonprimitive paths fail. Disabled bloom columns remain disabled. + +| Option | Default | Meaning | +|---|---|---| +| compressionCodec | SNAPPY | Codec name/CompressionCodecName; tested SNAPPY, ZSTD, GZIP, UNCOMPRESSED | +| inferPrecisionAndScale | true | Infer exact decimal settings when no explicit settings apply | +| precision / scale | null / null | Uniform positive precision and nonnegative scale, provided together | +| decimalMeta | empty | Logical field path to `[precision, scale]`; overrides uniform settings | +| roundingMode | UNNECESSARY | Reject lossy decimal rescaling; opt in to e.g. HALF_UP deliberately | +| zoneId | system zone (null) | LocalDateTime to adjusted timestamp conversion | +| rowGroupSize | 134217728 | Positive target row-group bytes | +| pageSize | 1048576 | Positive target data-page bytes | +| dictionaryEncoding | true | Dictionary encoding with upstream fallback | +| writerVersion | V1 | V1 or V2 writer encodings and page layout | +| columnIndexTruncateLength | 64 | Positive binary page-bound truncation bytes | +| statisticsTruncateLength | 2147483647 | Positive row-group binary statistics truncation bytes | +| bloomFilterEnabled | empty, disabled | Primitive path to Boolean override | +| bloomFilterNDV | empty | Primitive path to positive expected distinct-value count | +| sizeStatisticsEnabled | true | Collect size statistics | + +Strings are STRING-annotated recursively; byte[] stays raw BINARY. Byte/Short use signed width +annotations, UUID uses FIXED[16], LocalTime uses TIME(NANOS), and Instant uses adjusted +TIMESTAMP(MICROS). Instants with sub-microsecond precision or int64-microsecond overflow fail; +Instant.MIN/MAX are unsupported. Character and enums become String (`name()` for enums). +ZonedDateTime/OffsetDateTime/custom objects require explicit conversion to Instant/String/Map. +Date/Timestamp subclasses must share a single encoding policy; mixed units fail before output. + +Object and abstract/interface declarations scan every non-null value, check assignability and +infer one encoding; canonical metadata replaces the supertype. Concrete declarations require +instance-compatible values and a single runtime encoding policy. Mixed numeric classes need +explicit normalization. All-null Object becomes String; all-null Number/Temporal requires an +explicit supported concrete type. Containers use canonical List/Map metadata. All supported +metadata names are allowlisted; unknown external class names fall back to schema inference. + +Every resolved BigDecimal uses exact DECIMAL FIXED storage. Settings precedence is per-field, +uniform, then inference. Nested logical paths use `.element`, `.key`, `.value`, and struct field +names (for example `nested.element.amount`). With inference disabled every decimal needs settings. +All-null BigDecimal infers `(1, 0)`. There is no DOUBLE fallback. Explicit decimal conversion obeys +roundingMode. See [migration notes](release.md) for changed 0.7.x behavior. + +## CDC output, tests and benchmarks + +Native CDC writing is deferred: [parquet-java #3818](https://github.com/apache/parquet-java/pull/3818) +was open/unmerged when checked on 2026-10-17. Tuning pages/row groups is not CDC. For CDC output, +rewrite with PyArrow and preserve metadata: + +```bash +python3 -m venv /tmp/matrix-parquet-fixtures +/tmp/matrix-parquet-fixtures/bin/python -m pip install pyarrow==21.0.0 +/tmp/matrix-parquet-fixtures/bin/python - <<'PY' +import pyarrow.parquet as pq +table = pq.read_table('prices.parquet') +pq.write_table(table, 'prices-cdc.parquet', compression='zstd', + use_content_defined_chunking=True, write_page_index=True) +assert pq.read_table('prices-cdc.parquet').equals(table, check_metadata=True) +PY ``` -**Important:** For consistent round-trip behavior, use the same timezone for writing and reading. If files are shared across systems with different default timezones, explicitly specify the timezone. - -`java.sql.Time` is different: it is always treated as a local time of day in the JVM default timezone, independently of `zoneId`, and preserves milliseconds. External `TIME(MICROS)` and `TIME(NANOS)` values are read as `Time` with their sub-millisecond fraction truncated; the Parquet `isAdjustedToUTC` flag does not cause a zone shift. - -## Compression - -Parquet files are compressed with `SNAPPY` by default. Override per write: - -```groovy -import org.apache.parquet.hadoop.metadata.CompressionCodecName - -// Builder API -MatrixParquetWriter.builder(data) - .compressionCodec(CompressionCodecName.GZIP) - .write(file) - -// String form (also accepted) -MatrixParquetWriter.builder(data) - .compressionCodec('ZSTD') - .write(file) - -// SPI options map -data.write([compressionCodec: 'UNCOMPRESSED'], file) +This loads the whole Arrow table and adds a rewrite pass. Python is not a runtime dependency. +Hadoop filesystem configuration is avoided for local reads/writes; upstream codec initialization +still requires Hadoop classes, so the dependency and JDK 21 limit remain. + +```bash +./gradlew :matrix-parquet:codenarcMain :matrix-parquet:codenarcTest +./gradlew :matrix-parquet:spotlessCheck +./gradlew :matrix-parquet:test :matrix-parquet:compileConsumerApi +PARQUET_PYTHON=/tmp/matrix-parquet-fixtures/bin/python ./gradlew :matrix-parquet:test -PrunSlowTests=true +/tmp/matrix-parquet-fixtures/bin/python matrix-parquet/src/test/python/verify_matrix_output.py +./gradlew :matrix-parquet:test --tests ParquetInteropTest --rerun-tasks +./gradlew test ``` -Supported codec names: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `ZSTD`, `LZ4_RAW` (any value of `org.apache.parquet.hadoop.metadata.CompressionCodecName`). Compression is transparent on read — `MatrixParquetReader` auto-detects the codec from the file footer, no reader-side configuration is needed. - -## Known Limitations - -### Memory Usage -Matrix is an **in-memory data structure**. The entire Parquet file is loaded into memory when reading. Ensure you have sufficient RAM available for: -- The raw data size -- Java object overhead (typically 2-3x the raw data size) -- Working memory for processing - -For very large datasets that don't fit in memory, consider: -- Processing files in chunks externally before loading -- Using dedicated big data tools (Spark, Hadoop) for initial filtering -- Increasing JVM heap size (`-Xmx`) - -### JDK Compatibility -This module requires **JDK 21** and is limited to JDK 21 maximum due to Hadoop 3.4.x compatibility constraints. JDK 22+ is not supported. - -### Timestamp Precision -- `LocalDateTime` and `Timestamp` are stored with **microsecond** precision -- `Time` is stored with **millisecond** precision -- Nanosecond precision is not preserved - -### Type Metadata -Column type information is stored in Parquet file metadata. When reading files not created by `MatrixParquetWriter`: -- Types are inferred from Parquet schema logical type annotations -- Some type information may be lost (e.g., `java.sql.Timestamp` vs `LocalDateTime`) -- `BigInteger` columns read without Matrix metadata (e.g. external files) are inferred as `BigDecimal` with scale 0, since both the Parquet schema and the inference logic only see a generic `DECIMAL` logical annotation -- `BigInteger` values nested inside `List`/`Map` columns always get a declared precision of 38 regardless of their actual digit count (top-level column precision inference does not extend into nested structures), and round-trip back as numerically-equal `BigDecimal` values rather than `BigInteger` (nested elements carry no expected-type metadata, unlike top-level columns). This does not cause data loss or truncation, but external Parquet readers that validate `DECIMAL` precision against declared metadata rather than actual byte length may reject or misinterpret values with more than 38 digits - -## Technical Notes - -### Thread Safety -Both `MatrixParquetWriter` and `MatrixParquetReader` use thread-safe caching for: -- PropertyDescriptor lookups (for struct handling) -- Class.forName() results (for type parsing) - -The timezone parameter uses thread-local storage, making concurrent writes/reads with different timezones safe. - -### Performance Tips -1. Use `inferPrecisionAndScale=true` (default) for automatic BigDecimal handling -2. For large matrices with struct columns, the first write may be slower due to reflection caching -3. Subsequent operations on the same struct types benefit from cached metadata - -## 3rd Party Libraries - -| Library | Purpose | License | -|---------|---------|---------| -| org.apache.parquet:parquet-column | Parquet columnar format | Apache 2.0 | -| org.apache.parquet:parquet-hadoop | Hadoop integration | Apache 2.0 | -| org.apache.hadoop:hadoop-common | Hadoop utilities | Apache 2.0 | -| org.apache.hadoop:hadoop-mapreduce-client-core | MapReduce client | Apache 2.0 | +Fixture reproduction: run `matrix-parquet/src/test/python/generate_optimized.py` from the repository root with +pinned PyArrow. Checked-in Parquet fixtures total less than 4 MiB (each less than 1 MiB). +Default-size CDC is generated in `build/default-cdc` only in the slow lane; missing Python/Arrow +produces an explicit skip. CI provisions pinned Arrow and checks fixture reproducibility. +See [measurements and reproduction](benchmarks/optimized-parquet.md) for the opt-in benchmark. diff --git a/matrix-parquet/release.md b/matrix-parquet/release.md index 59fd65be9..5381c3dfd 100644 --- a/matrix-parquet/release.md +++ b/matrix-parquet/release.md @@ -1,5 +1,56 @@ # Matrix-parquet Release History +## v0.8.0 (unreleased) + +Add CDC/page-index interoperability fixtures; a column-oriented row-group reader; typed predicates, +projection, limits and closeable batches; strict sharded datasets with bounded parallel decoding; +HTTP range reads; Hub dataset URI resolution; and shared writer tuning. Local files and byte arrays +use InputFile directly. New files carry `matrix.parquet.version=2`. + +Complete defaults, supported types and end-to-end examples are in [the module guide](readme.md). + +| Behavior | 0.8.0 result | Compatibility/opt-out | +|---|---|---| +| New Matrix strings | STRING-annotated BINARY recursively | No opt-out; standard annotation fixes interoperability | +| Legacy Matrix plain BINARY text | Remains String, including nested text/map keys | Automatic; metadata carrying the new format marker overrides legacy inference | +| 0.7.x byte[] columns | Fail with a contextual unrecoverable-legacy-binary error | The pinned 0.7.x writer fails with InvalidSchemaException for byte[]; legacy `[B` text payloads cannot recover bytes; regenerate from source | +| External raw BINARY | byte[] | `binaryAsString(true)` restores prior UTF-8 behavior | +| Unadjusted timestamps | Local wall-clock time preserved | No bug-preserving flag; applications needing the former shift can explicitly convert the local value through UTC into the desired zone | +| Integer-backed DECIMAL | BigDecimal with declared scale | No bug-preserving flag; explicit `unscaledValue()` obtains prior raw numeric value | +| UUID annotation | UUID | No text-decoding compatibility flag; explicit UUID/string/byte conversion at application boundary | +| FLOAT16 / INT96 unsupported cases | Contextual unsupported-type error | No UTF-8 fallback; convert these types explicitly before reading | +| Byte / Short fallback | Signed INT32 with INT(8/16), restored as Byte/Short | Legacy unannotated text stays String; explicitly convert to String before writing to retain text output | +| LocalTime fallback | TIME(NANOS), restored as LocalTime using new-format metadata | Legacy text stays String; explicit String conversion retains text output | +| Instant fallback | TIMESTAMP(MICROS, adjusted=true), restored as Instant using new-format metadata | Legacy text stays String; reject sub-microsecond precision loss and int64-micros overflow; explicitly convert to String for text output | +| UUID writer fallback | FIXED[16] + UUID, restored as UUID | Legacy text stays String; explicit String conversion retains text output | +| Character fallback | STRING, returned as String | Normalize emitted metadata to String; Character identity is not preserved | +| Enum fallback | STRING using enum name(), returned as String | Normalize metadata to String; convert explicitly for former overridden toString() output | +| ZonedDateTime / OffsetDateTime fallback | Explicit unsupported-type error | Convert to Instant to retain instant, or String to retain zone/offset text; no implicit zone loss | +| Object column, single supported policy type | That type's encoding and canonical metadata | Reads back typed as the inferred canonical class instead of Object; no explicit types required | +| Abstract/interface declared supertype (Number, CharSequence, Comparable, Serializable, Temporal) | Validate assignability, infer one supported policy type and emit its canonical name | Reads back as the resolved class instead of the declared supertype; normalize heterogeneous values explicitly | +| Concrete declared type with incompatible runtime values | Validation error before writing | Box primitives then require every non-null value to be an instance; no silent schema/metadata mismatch | +| BigDecimal, declared or inferred through a supertype | Exact FIXED_LEN_BYTE_ARRAY DECIMAL with resolved precision/scale | Remove DOUBLE fallback; infer by default, or require explicit decimal settings when inference is disabled; no lossy compatibility mode | +| All-null Number/Temporal declaration | Validation error before writing | New write failure: retype explicitly to a supported concrete type; no nominal String metadata | +| Concrete Date column mixing Date/Timestamp or sql.Date/Timestamp | Validation error before writing | Runtime encoding policies differ; normalize every value to one supported temporal class before writing | +| All-null Object column | STRING/BINARY with canonical String metadata | Retain current text schema behavior; reads back as String-typed column | +| Mixed/unsupported Object column or other custom types | Contextual validation error before writing | Name the first offending row/class; explicitly normalize values or map to String | + +Unknown metadata class names are never loaded and fall back to physical/logical schema inference. +Incompatible metadata declarations also fall back rather than creating falsely typed columns. +Legacy Matrix text compatibility applies recursively; new raw binary metadata remains authoritative. +The checked-in original 0.7.x string fixture predates these writer changes. Legacy byte[] recovery +claims were narrowed after an end-to-end check found InvalidSchemaException in this checkout. + +External INT96/FLOAT16 raise contextual unsupported-type errors. External TIME still returns +java.sql.Time at millisecond precision; exact predicate filtering preserves original precision. +Mixed Date/Timestamp policies are now rejected before writing, preventing the prior 1000× unit +corruption. New Object/supertype values are resolved consistently for schema and metadata. +Unknown SPI options are now rejected. Dataset reads omit Matrix indices, require matching type +metadata, and reject empty selections and duplicate paths. + +Native CDC writing remains deferred to upstream parquet-java #3818; the Python rewrite preserves +custom metadata. Hadoop stays as a runtime dependency because codec initialization still loads it. + ## v0.7.0, 2026-09-22 - SPI `inferPrecisionAndScale` parsing is now strict: Boolean values and trimmed, case-insensitive `true`/`false` strings are accepted; former truthy inputs are rejected. - Breaking change: explicit DECIMAL scale conversion now defaults to `RoundingMode.UNNECESSARY`, rejecting lossy conversion with a contextual `IllegalArgumentException`. Set `roundingMode: 'HALF_UP'` (or another named `RoundingMode`) when rounding is intentional. diff --git a/matrix-parquet/req/optimized-parquet-plan.md b/matrix-parquet/req/optimized-parquet-plan.md new file mode 100644 index 000000000..4228d2b72 --- /dev/null +++ b/matrix-parquet/req/optimized-parquet-plan.md @@ -0,0 +1,782 @@ +# Hugging Face optimized Parquet: analysis and enhancement plan + +Analysis date: 2026-10-17. Scope: compatibility and baseline measurements (P0), columnar and bounded local reads, +typed predicates and writer tuning (P1), sharded datasets and HTTP ranges (P2), +optional Hub resolution and upstream CDC tracking (P3). Implementation and verification evidence is recorded at the end of this document. +The findings below describe the pre-implementation baseline. + +## Findings + +### What “optimized Parquet” means + +Hugging Face describes optimized Parquet as ordinary Parquet written with content-defined +chunking (CDC) and page indexes. These are writer/layout features, not a separate container +format, codec, file extension, or reader mode. PyArrow exposes +`use_content_defined_chunking=True` and `write_page_index=True`; its CDC option requires +PyArrow 21 or later. [Hugging Face documentation](https://huggingface.co/docs/hub/en/datasets-libraries), +[PyArrow integration](https://huggingface.co/docs/hub/en/datasets-pyarrow). + +CDC chooses boundaries from logical column content before encoding/compression. Readers +decode the resulting ordinary pages without reproducing the chunking algorithm. Xet can +reuse matching stored chunks across revisions; reading a downloaded local file does not +itself invoke Xet or give matrix-parquet incremental download capability. +[Hugging Face CDC explanation](https://huggingface.co/blog/parquet-cdc). + +Page indexes consist of `ColumnIndex` (page statistics) and `OffsetIndex` (page locations +and first-row positions), associated with column chunks. They support skipping candidate +pages and aligning reads across columns. Indexes are optional, and page boundaries need +not align across columns. Filtering must still reject nonmatching records within candidate +pages. [Parquet page-index specification](https://parquet.apache.org/docs/file-format/pageindex/). + +PyArrow can move page statistics from page headers to the index when index writing is +enabled. Thus tests must include real PyArrow output rather than assuming that Java-written +round trips cover this layout. CDC and page indexes are independent switches; test all four +combinations. Do not assume fixed page sizes or infer CDC from a page-size distribution. +[PyArrow writer reference](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.write_table.html). + +### Pre-implementation repository behavior + +The authoritative dependencies are **parquet-java 1.18.1**, Hadoop **3.5.0** in +`gradle/libs.versions.toml`; the module is **0.7.1-SNAPSHOT**. The older v0.6.0 plan is +historical and should not determine dependency decisions here. Keep JDK 21. + +| Area | Evidence in current code | Assessment | +|---|---|---| +| Full-file decoding | `MatrixParquetReader.read(File, ...)` reads the footer and uses `ParquetReader` / `GroupReadSupport` | CDC/indexed files should decode through the existing dependency; unproven until external fixtures run | +| Selective reads | `ParquetReadOptions` and `ReaderBuilder` only expose name/time zone; core loop materializes every row and field | No caller-supplied predicate, projection, or bounded read; no useful page pruning for the current full scan | +| Remote reads | URL overloads open a stream; `readFromInputStream` copies it entirely to a temporary file | Entire file transfers before reading; page pruning cannot reduce network traffic | +| Metadata | Reader uses Matrix-specific type/index keys when present, otherwise schema inference | External files need tests without Matrix metadata; Matrix indexes are unrelated to Parquet page indexes | +| Writes | File and memory writers configure schema, codec and Matrix metadata | Index production is delegated to upstream defaults; no CDC configuration | +| Coverage | Six Groovy test classes and `mtcars.parquet` fixture; no CDC/index-specific suite | Cross-implementation and page-pruning coverage missing | + +parquet-java 1.18.1 already exposes `withFilter`, `useColumnIndexFilter(boolean)`, and +record filtering. Column-index filtering is enabled by default by its read-options builder. +The gap is supplying a predicate and preserving correct materialization, rather than merely +turning an upstream flag on. +[Reader API](https://github.com/apache/parquet-java/blob/apache-parquet-1.18.1/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetReader.java), +[read defaults](https://github.com/apache/parquet-java/blob/apache-parquet-1.18.1/parquet-hadoop/src/main/java/org/apache/parquet/ParquetReadOptions.java). + +The upstream writer enables statistics by default and serializes column/offset indexes. +Consequently, “add page-index writing” is not yet an established missing feature: first +inspect actual matrix-parquet output for supported columns. There is no CDC option in the +examined 1.18.1 `ParquetWriter.Builder` or `ParquetProperties.Builder`. Do not introduce a +nonfunctional `optimizedParquet` or `contentDefinedChunking` option. +[Writer API](https://github.com/apache/parquet-java/blob/apache-parquet-1.18.1/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java), +[writer properties](https://github.com/apache/parquet-java/blob/apache-parquet-1.18.1/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java), +[index serialization](https://github.com/apache/parquet-java/blob/apache-parquet-1.18.1/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java). + +### Public API and dependency boundary + +`parquet-column` and `parquet-hadoop` are `implementation` dependencies. Exposing +`FilterPredicate` would force downstream consumers to add parquet-java to their compile +classpath or require changing publication scope to `api`. Use Matrix-owned predicate and +writer-version types instead; translate them internally. Keep parquet-java implementation +private for the new API and add downstream Gradle/Maven compile smoke tests. Existing public +codec signatures are a separate legacy exposure to characterize, not a reason to introduce +more upstream types. + +### Adjacent schema risks + +These affect Hugging Face datasets independently of optimization and must be tracked +separately from CDC compatibility: + +- Unannotated `BINARY`/`FIXED_LEN_BYTE_ARRAY` currently defaults to `String`, and + `readPrimitive` decodes it as UTF-8. Arbitrary image/audio bytes therefore lose fidelity. +- DECIMAL annotations on `INT32`/`INT64` are inferred as `BigDecimal`, but the corresponding + primitive read branches return raw integers without applying decimal scale. +- TIMESTAMP decoding ignores `isAdjustedToUTC=false` and shifts local wall-clock values + through the configured/system zone. Test naive PyArrow timestamps explicitly before declaring + their annotation confirmed by fixtures. +- UUID and FLOAT16 annotations on FIXED_LEN_BYTE_ARRAY reach the UTF-8 fallback. INT96 + reaches `group.getString`; the exact exception remains to be characterized by a fixture. Neither + path constitutes valid decoding. +- `readList` expects the standard three-level `list` structure. Include PyArrow compliant + lists, nested structs and null/empty values; do not claim support for all historical LIST + representations without additional work. + +Prefer a small representative text dataset as the initial compatibility target. Include +binary/media and integer-backed decimal fixtures to make broader limitations explicit. + +## Implementation sequence + +### 0. Set the release and migration contract (P0 prerequisite) + +- 0.1 [x] Before production changes, set `matrix-parquet/build.gradle` to `0.8.0-SNAPSHOT`; target + release **0.8.0**. Keep P2/P3 follow-ups separately releasable rather than promising their + delivery in 0.8.0. Add a 0.8.0 migration section to release notes with the behavior table below, + complete option defaults and runnable examples. + +- 0.2 [x] Capture representative 0.7.x-produced fixtures before changing the writer, including + top-level/nested strings, string map keys and Matrix temporal/decimal metadata. Include a 0.7.x + byte[] failure fixture; do not assume every old file is recoverable. Include them in the 4 MiB + fixture budget. Add before/after characterization tests plus external fixtures for behavior + 0.7.x did not write, asserting values and runtime types. The Group reader retained for + differential tests must receive phase-2 fixes; separately preserve original expected 0.7.x + results for migration characterization. + + | Behavior | 0.8.0 result | Compatibility/opt-out | + |---|---|---| + | New Matrix strings | STRING-annotated BINARY recursively | No opt-out; standard annotation fixes interoperability | + | Legacy Matrix plain BINARY text | Remains String, including nested text/map keys | Automatic; metadata carrying the new format marker overrides legacy inference | + | 0.7.x byte[] columns | Fail with a contextual unrecoverable-legacy-binary error | The pinned original writer fails with InvalidSchemaException for byte[]; legacy `[B` text cannot recover bytes; regenerate from source | + | External raw BINARY | byte[] | `binaryAsString(true)` restores prior UTF-8 behavior | + | Unadjusted timestamps | Local wall-clock time preserved | No bug-preserving flag; applications needing the former shift can explicitly convert the local value through UTC into the desired zone | + | Integer-backed DECIMAL | BigDecimal with declared scale | No bug-preserving flag; explicit `unscaledValue()` obtains prior raw numeric value | + | UUID annotation | UUID | No text-decoding compatibility flag; explicit UUID/string/byte conversion at application boundary | + | FLOAT16 / INT96 unsupported cases | Contextual unsupported-type error | No UTF-8 fallback; supported-type contract fixed before release | + | Byte / Short fallback | Signed INT32 with INT(8/16), restored as Byte/Short | Legacy unannotated text stays String; explicitly convert to String before writing to retain text output | + | LocalTime fallback | TIME(NANOS), restored as LocalTime using new-format metadata | Legacy text stays String; explicit String conversion retains text output | + | Instant fallback | TIMESTAMP(MICROS, adjusted=true), restored as Instant using new-format metadata | Legacy text stays String; reject sub-microsecond precision loss and int64-micros overflow; explicitly convert to String for text output | + | UUID writer fallback | FIXED[16] + UUID, restored as UUID | Legacy text stays String; explicit String conversion retains text output | + | Character fallback | STRING, returned as String | Normalize emitted metadata to String; Character identity is not preserved | + | Enum fallback | STRING using enum name(), returned as String | Normalize metadata to String; convert explicitly for former overridden toString() output | + | ZonedDateTime / OffsetDateTime fallback | Explicit unsupported-type error | Convert to Instant to retain instant, or String to retain zone/offset text; no implicit zone loss | + | Object column, single supported policy type | That type's encoding and canonical metadata | Reads back typed as the inferred canonical class instead of Object; no explicit types required | + | Abstract/interface declared supertype (Number, CharSequence, Comparable, Serializable, Temporal) | Validate assignability, infer one supported policy type and emit its canonical name | Reads back as the resolved class instead of the declared supertype; normalize heterogeneous values explicitly | + | Concrete declared type with incompatible runtime values | Validation error before writing | Box primitives then require every non-null value to be an instance; no silent schema/metadata mismatch | + | BigDecimal, declared or inferred through a supertype | Exact FIXED_LEN_BYTE_ARRAY DECIMAL with resolved precision/scale | Remove DOUBLE fallback; infer by default, or require explicit decimal settings when inference is disabled; no lossy compatibility mode | + | All-null Number/Temporal declaration | Validation error before writing | New write failure: retype explicitly to a supported concrete type; no nominal String metadata | + | Concrete Date column mixing Date/Timestamp or sql.Date/Timestamp | Validation error before writing | Runtime encoding policies differ; normalize every value to one supported temporal class before writing | + | All-null Object column | STRING/BINARY with canonical String metadata | Retain current text schema behavior; reads back as String-typed column | + | Mixed/unsupported Object column or other custom types | Contextual validation error before writing | Name the first offending row/class; explicitly normalize values or map to String | + +### 1. Establish compatibility and writer-index facts (P0, first deliverable) + +- 1.1 [x] Add `OptimizedParquetTest.groovy` and checked-in deterministic fixtures + under `src/test/resources/optimized/`. Add a generator script plus a manifest containing + generator/PyArrow version, options, schema, row counts, expected values and SHA-256 checksums. + Python is a fixture-generation tool, not a runtime or normal Gradle-test dependency. + +- 1.2 [x] Generate paired ordinary/CDC-only/index-only/CDC+index files from identical values. + Include SNAPPY and ZSTD, dictionary enabled/disabled, data pages v1/v2, multiple pages per + column and multiple row groups. Use deterministic varied long strings so the CDC fixtures + actually cross chunk boundaries; verify physical multi-page structure. Set a **4 MiB aggregate + checked-in binary-fixture budget**, with a 1 MiB per-file cap. Use CDC `{'min_chunk_size': 4096, + 'max_chunk_size': 16384, 'norm_level': 0}` for small fixtures, and verify the resulting + boundaries/pages. PyArrow 21 accepts this dictionary with both size keys required, confirmed in + its [versioned implementation](https://github.com/apache/arrow/blob/apache-arrow-21.0.0/python/pyarrow/_parquet.pyx). + Add an opt-in default-CDC generation/interoperability test under `./gradlew :matrix-parquet:test + -PrunSlowTests=true`, using a build-directory fixture. Skip with an explicit reason if Python or + pinned PyArrow is absent; do not silently install packages. Add the Python/PyArrow lane to + `.github/workflows/release-all-tools.yml` (including its pull_request runs), invoking the opt-in + command so this case actually runs. Fail generation errors when dependencies are available and + enforce the checked-in budget. Pairwise coverage replaces a Cartesian product. Include + ordered/unordered columns, null-only pages, null/empty nested lists and structs, Unicode, + decimals and timestamps. + +- 1.3 [x] Assert exact values, types, order and null semantics through file, path, bytes, stream, + builder and SPI entry points. Use a local HTTP server for URL paths; regular tests must not + depend on Hugging Face availability or credentials. Cover empty files and external metadata + without Matrix-specific keys. + +- 1.4 [x] Inspect index references and deserialize `ColumnIndex`/`OffsetIndex` using + `ParquetFileReader`. Check page counts, monotonic offsets, row positions and valid bounds. + Inspect both Java-produced file and byte-array output. Record which types produce indexes and + any upstream omissions. Do not infer index presence from a flag. + +- 1.5 [x] Repair any demonstrated CDC/index decoding failure with a targeted regression test. If + all supported-schema fixtures pass, record that no CDC decoder change is necessary. Do not + upgrade parquet-java solely because the files use CDC. + +- 1.6 [x] Before reader changes, establish a repeatable full-scan baseline using generated + approximately 1M-row wide-text, low-cardinality categorical and mixed/nested datasets. Keep + large inputs outside Git. Record generation commands/seeds, JVM/heap/GC, CPU, warmup and + repeated-run medians, rows/s, elapsed time, allocation/GC, peak heap and bytes read. Separate + cold and warm I/O and compare ordinary/indexed/CDC+indexed layouts. Reuse these exact inputs + throughout P1/P2; include narrow projection and selective predicates as soon as those APIs + exist. + +### 2. Fix demonstrated external-schema fidelity gaps (P0 for broad HF support) + +Order within 2.0: **2.0 → 2.0c (shared descriptor) → 2.0a/2.0b/2.0d +(policies implemented on that descriptor) → 2.0e (integration verification)**. +Task identifiers remain stable for review references; their numeric order does not override +these dependencies. Do not implement policies on the old first-value inference as an interim step. + +- 2.0 [x] Before 2.1, write String as BINARY + `LogicalTypeAnnotation.stringType()` through the + shared primitive schema builder, including list elements, struct fields, map values and required + map keys. Verify the annotations with PyArrow. Preserve existing files: when Matrix metadata + identifies a legacy file, unannotated BINARY remains String recursively, including nested values + whose expected type is null. New binary-capable output must include a versioned Matrix + schema/encoding marker to distinguish raw binary from legacy nested text; do not apply legacy + rules blindly to newly written media bytes. Add nullable Boolean `binaryAsString` (default + null/AUTO): null uses legacy-Matrix text compatibility and raw bytes for external/new-format + files; true forces unannotated BINARY to UTF-8 strings recursively; false forces raw bytes + recursively. Annotations take precedence; explicitly declared Matrix String fields remain + strings. Define FIXED binary separately: never override DECIMAL/UUID/FLOAT16 annotations. + Propagate through options, builder and SPI, and test old nested strings/map keys plus new mixed + binary/text fields. + +- 2.0a [x] Require raw byte[] writing as BINARY without STRING annotation, with the new format + marker, at top level and inside nested structures. Add an explicit byte[] branch using Parquet + Binary bytes; reject unsupported schema/value combinations with contextual errors instead of the + `value.toString()` fallback only after implementing the explicit type policies in 2.0b–2.0d. Land + 2.0a–2.0e together so fallback removal has no implicit scope. Resolve metadata array class names + through a fixed allowlist mapping of names to supported writer-produced types, including `[B` to + byte[]. Do not load arbitrary classes named by untrusted file metadata. If a resolver still + needs `Class.forName(name, false, loader)`, call it only for allowlisted names. Unknown names + fall back to per-field schema inference, without failing the whole read; malformed metadata must + not shift type/column alignment. Test unknown/custom class names and arrays. The legacy `[B` + corruption rule below takes precedence over both allowlist resolution and schema fallback. Add + file/bytes/stream/SPI round-trip tests for null, empty and arbitrary non-UTF-8 bytes, plus + PyArrow verification of exact payloads. Characterize original byte[] write failure and the `[B` + class-resolution failure. Use explicitly synthetic legacy text metadata if the original writer + cannot produce a valid file; fixing class lookup cannot recover a textual payload. Fail explicitly on legacy Matrix byte[] metadata rather than returning the + textual payload as if it were original binary. State this in migration notes and test it; + marker-aware new byte[] files must read normally. + +- 2.0b [x] Implement per-type encodings and explicit rejections. Replace remaining implicit + `toString()` cases with the policies in 0.2. Implement Byte/Short as signed INT32 + + `intType(8/16, true)`, with range validation and annotated nested-width inference. Encode LocalTime as + TIME(NANOS) to preserve nanoseconds, and Instant as adjusted TIMESTAMP(MICROS). Reject Instant + values with sub-microsecond precision rather than silently truncating, and validate int64 + epoch-micros overflow using checked arithmetic, including negative/pre-epoch instants. Test + historical dates beyond the int64-nanos range, exact microseconds, fractional-microsecond + rejection and extreme Instant bounds. MICROS has a wider range than NANOS but still cannot + represent Instant.MIN/MAX; do not promise those. No NANOS write option is required in this + release. New-format metadata restores these exact Java types; document external schema-inference + types separately and preserve existing java.sql.Time/LocalDateTime contracts. Encode UUID as 16 + bytes with `uuidType()` in the specification's byte order. Character becomes STRING; enums become + STRING using `name()`, with canonical String metadata at every supported nesting level. Reject + ZonedDateTime, OffsetDateTime and remaining custom classes before writing rows; messages describe + explicit Instant/String conversion or column retyping. + +- 2.0c [x] Unify column type resolution and validation. Resolve every column through one validated + schema/type descriptor used for physical schema, value encoding, decimal settings and emitted + metadata. For Object and abstract/interface supertypes (Number, CharSequence, Comparable, + Serializable, Temporal), scan all non-null values, require assignability to the boxed declaration + and one supported policy type, then emit that encoding's canonical name. Never decide from only + the first sample. For concrete declarations, including non-final concrete classes, require each + non-null value to be an instance after primitive boxing normalization; additionally validate the + single encoding policy across subclasses. A non-final modifier alone does not turn a concrete + declaration into an inference-only supertype. Report the first offending zero-based row, declared + type, runtime class and expected policy before output mutation. Resolve supported concrete + all-null columns from their declaration; Object or abstract/interface all-null columns use + canonical String only if compatible with the declaration, otherwise require an explicit supported + type. Thus all-null Number/Temporal columns need explicit retyping rather than false String + metadata. For supported concrete declarations, the single encoding policy comes from the runtime + values, while metadata retains the declared supported type unless an explicit canonicalizing + policy (Character/Enum) applies. Empty columns use the declared policy. Mixed Date/Timestamp + values fail even though Timestamp is assignable to Date, because MILLIS/MICROS policies differ; + never dispatch incompatible units per value. Add a legacy characterization test for Date first, + Timestamp later, recording the current 1000× timestamp corruption, and sql.Date first, Timestamp + later, recording the current write failure. New tests require rejection before output mutation in + either order, plus homogeneous temporal-subclass round trips with declared Date metadata. Test + Object/supertype inference, concrete mismatch, boxed normalization and all-null rules separately. + Test Number/CharSequence/Comparable/Serializable/Temporal declarations, and reject a concrete + BigDecimal column containing Integer values before writing. + +- 2.0d [x] Route every resolved BigDecimal column, declared or resolved from Object/Number/other + supertypes, through shared `inferDecimalPrecisionAndScale` and column-name `decimalMeta` handling + before schema construction. Use explicit per-column settings first, explicit global + precision/scale next, then inference if enabled. With `inferPrecisionAndScale(false)`, require + complete explicit settings for each resolved BigDecimal field; reject missing settings rather than + falling back to DOUBLE. Apply existing rounding/precision validation to all these paths. Always + emit FIXED_LEN_BYTE_ARRAY + DECIMAL and canonical BigDecimal metadata together; remove the DOUBLE + fallback for declared BigDecimal too. Handle nested decimals with the same resolved-field policy; + document nested metadata addressing and reject unresolved settings. Use a documented minimum + precision/scale (1, 0) for all-null declared BigDecimal when inferring. Test untyped Groovy + literals `[1.5, 2.25]`, high-precision values that DOUBLE cannot represent, negative scales, + Number-declared decimals, per-column overrides and inference disabled with and without explicit + settings. Assert exact BigDecimal values, runtime classes, declared Matrix types, physical DECIMAL + metadata and all-null behavior across entry points. General declared-type validation tests belong + to 2.0c. Canonical metadata must never claim BigDecimal over a DOUBLE or INT32 encoding. + +- 2.0e [x] Verify metadata coverage, migration contracts and cross-entry-point behavior. Write the + canonical type name instead of Object or the abstract/interface declaration. Canonicalized + Character/Enum values follow the String policy; do not silently widen heterogeneous numeric types. + Reject mixed or unsupported policy types with the first offending row index (document zero-based), + actual class and expected policy. All-null Object columns retain STRING/BINARY with canonical + String metadata. Validate nested inferred schemas consistently before writing. Do not discover + unsupported types halfway through overwriting an output file. Include null/all-null columns and + boxed primitive normalization in schema validation. Add ordinary/optimized and file/bytes/SPI + round-trip tests for each supported policy, nested variants where supported, and contextual + rejection tests for each rejected class. Characterize legacy text separately: it remains String, + not reconstructed Java objects. Assert writer-emitted matrix.columnTypes names are a subset of the + resolver allowlist across all supported types and canonicalized columns, including Byte/Short and + containers. Never emit java.lang.Object or arbitrary enum/custom class names in new files; old + unknown names continue to use the schema fallback. Test the untyped builder example with + Integer/String data, homogeneous nullable Object columns, all-null Object columns and mixed + `[1, 'x']` rejection before output mutation. Assert inferred canonical metadata names and resulting + Matrix types, including Object columns of each supported policy type. Extend migration notes with + each 0.2 row. For each of 2.0a–2.0e, run the ordered module checks in 6.3 and record the + successful commands/results before checking off that task; do not use a single completion checkbox + for all five areas. + +- 2.1 [x] Add byte-preserving inference/conversion for raw binary fields, including binary inside + nested media structs. Retain string decoding for STRING annotations and DECIMAL handling and the + 2.0 legacy/override rules. Document the compatibility impact of changing external raw-binary + columns from String to byte[]. Binary writer/reader round-trip support in 2.0a is mandatory, not + conditional. + +- 2.2 [x] Apply DECIMAL scale to INT32/INT64 physical values, including nested values; test + negative, null, scale-zero and fractional values against PyArrow expectations. + +- 2.3 [x] Respect TIMESTAMP `isAdjustedToUTC`: adjusted timestamps use the configured zone; + unadjusted timestamps retain local wall-clock fields via UTC epoch arithmetic + (`LocalDateTime.ofEpochSecond(instant.epochSecond, instant.nano, ZoneOffset.UTC)`). Cover + MILLIS/MICROS/NANOS, pre-epoch values, nested timestamps, DST boundaries and + UTC/Europe-Stockholm/America-New_York. Generate naive and timezone-aware PyArrow fields and + assert their footer annotations; preserve adjusted Matrix round trips. Specify Date/Timestamp + metadata overrides explicitly rather than silently assigning an instant to local timestamps. + +- 2.4 [x] Add UUID, FLOAT16 and INT96 fixtures. Decode UUID to `UUID`; define FLOAT16 widening to + `Float` with special-value tests, or fail explicitly if unsupported in the first delivery. + Characterize INT96 failure and implement documented legacy timestamp semantics or a contextual + unsupported-type error. Never silently decode these as text. Cover compliant, two-level and + legacy LIST layouts with explicit support/error contracts. + +- 2.5 [x] Keep these tests paired across ordinary and optimized layouts so failures are attributed + to schema conversion rather than CDC. Report unsupported logical types explicitly; broaden + beyond these cases only when a fixture demonstrates need. + +### 3. Expose selective local reads using upstream page filtering (P1) + +- 3.1 [x] Build a shared `InputFile` core using `LocalInputFile` and `PlainParquetConfiguration`; + use `LocalOutputFile`/`OutputFile` and plain configuration on file writes too. Preserve naming, + time-zone and ownership behavior across legacy, builder and SPI paths. Add an in-memory seekable + `InputFile` for byte arrays, replacing their temp-file copy. Test seeks, EOF and independent + streams without full-buffer copies. Use `ParquetFileReader.open(InputFile, ParquetReadOptions)` + and an explicit per-row-group loop, with `getRowGroups()`, + `readFilteredRowGroup(int)`/`readRowGroup(int)` and `MessageColumnIO.getRecordReader(pages, + materializer, filter)`. Set requested schema before filtering. Keep original footer group IDs + stable even if pruning removes groups from an internal list; spike and test this mapping before + exposing selectors. Close each `PageReadStore` on advancement/failure. Do not build the new core + around `ParquetReader`. Its 1.18.1 API does expose current-row-group index, but does not provide + direct indexed group selection; explicit groups fit batching and independent workers better. + Follow the upstream [internal reader loop](https://github.com/apache/parquet-java/blob/apache-parquet-1.18.1/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordReader.java), + without depending on that package-private class. Keep accept/rollback bookkeeping in the Matrix + materializer using the hooks in 3.2; the loop respects skipped/null records without duplicating + column-list truncation. Spike this integration before removing Hadoop paths. Test + SNAPPY/ZSTD/GZIP/uncompressed read/write with hadoop-common/mapreduce excluded, and inspect + linkage/dependency reports. Keep required codec classes until this proves removal safe; + distinguish avoiding filesystem initialization from removing dependencies. Do not raise the JDK + target as part of this change. + +- 3.2 [x] Replace `GroupReadSupport` with a custom `ReadSupport`/`RecordMaterializer` whose + converters append to typed column lists. Precompute schema/converter mappings and avoid per-cell + name lookup, per-row Group/list/Object[] creation and row-to-column transposes. Spike Matrix + construction from columns to verify ownership/copy behavior. Preserve nested list/map/struct + nulls, empty collections and parent-row alignment. Any active row filter, including ordinary + pushed-down record filtering, requires transactional output: converters may append values before + acceptance is known. Snapshot each output/filter-column list size at row start. Accept in Matrix + `RecordMaterializer.getCurrentRecord()`, which the filtering wrapper calls only after its + pushed-down predicate passes: evaluate any residual predicate there before accepting and return + null on rejection. Return a non-null lightweight record token for acceptance, including all-null + rows. Roll back to the snapshots in `skipCurrentRecord()`, reached through parquet-java's + skipped/null-record handling, and reset nested assembly state. Keep rollback idempotent. Mid-row + exceptions must invoke the same materializer-owned abort/rollback operation before propagating; + snapshots also protect this path. Test the actual hook ordering with pushed-down and residual + rejection. Test rejected first/middle/last rows and entire rejected groups, ensuring every + column has the same accepted-row count. Use staging where nested assembly requires it, without + allocating a generic row buffer for every unfiltered flat row. Differential-test against the old + reader before adding new projection/filter options; preserve schema fixes from phase 2 in shared + conversions. A ReadSupport adapter may initialize the converter tree, but the explicit + ParquetFileReader loop owns group selection and requested schema. + +- 3.3 [x] Implement dictionary-aware string reuse in converters: decode each dictionary ID once + per active dictionary and reuse the String reference, supporting dictionary changes and + plain-page fallback. No global `String.intern()` or unbounded cross-file cache. Verify identity + reuse on categorical data and exact values after dictionary fallback. + +- 3.4 [x] Add the proposed options below, including `toMap`, `fromMap`, descriptors, validation, + builder methods and public GroovyDoc. Use Matrix-owned immutable predicate types, translated + internally; no upstream predicate classes or closures in the new public contract. Reject unknown + columns, duplicates and incompatible predicate types before reading. Start projection with + top-level column names. + + | Proposed option | Type | Default | Meaning | + |---|---|---|---| + | `columns` | `List` | `null` | All columns; supplied nonempty list selects output columns in requested order | + | `filter` | Matrix-owned `ParquetPredicate` | `null` | Logical-schema predicate with exact record filtering | + | `useColumnIndexFilter` | `boolean` | `true` | Allow upstream page pruning when available; false is a comparison/diagnostic path | + +- 3.5 [x] Pass translated upstream predicates via `FilterCompat.get(predicate)`. The core loop + sets the union of output columns and all pushdown/residual filter columns via + `ParquetFileReader.setRequestedSchema(MessageType)` before filtering/group reads. ReadSupport + only initializes the matching converter/materializer tree; it does not own a second projection + implementation. Keep exact record filtering enabled. Read filter-only columns internally and + exclude them from the resulting Matrix. Add `Predicates.eq/ne/lt/le/gt/ge`, `isNull/isNotNull`, + `and/or/not` with typed value overloads for Number, String, Boolean, LocalDate, LocalDateTime, + Instant, LocalTime and byte[]. Convert dates, timestamp/time units, UTC adjustment, decimals + (including integer and fixed binary physical forms) using the actual file schema. Reject + overflow, lossy rescaling, incompatible local/instant comparisons and unsupported repeated-field + predicates. Implement exact logical residual evaluation when safe pushdown is unavailable; do + not incorrectly compare signed decimal bytes as generic lexicographic binary. Partition + pushdown/residual expressions safely, especially under OR/NOT. Document null/NaN semantics and + test them against full evaluation. + +- 3.6 [x] Remap `matrix.columnTypes` by original field name for projected/reordered columns. + Restore Matrix index metadata only when all indexed columns survive; otherwise omit that Matrix + index. Empty matches must still retain the selected schema and types. + +- 3.7 [x] Test result equivalence with index filtering on/off and indexes absent. Cover AND/OR, + null predicates, unordered data, truncated string bounds, boundary values and filter-only + columns. Treat unavailable indexes as a normal scan fallback; propagate genuine corruption + rather than silently returning partial results. + +- 3.8 [x] Prove page skipping using a counting seekable input and controlled multi-page fixture. + Disable row-group/dictionary/bloom pruning in this focused test so an entire row-group skip + cannot masquerade as page-index use. Require identical results and fewer data-page bytes read; + avoid timing assertions and exact read-call counts. + +- 3.9 [x] Add `limit(long)` (default unlimited/null, nonnegative; zero returns typed empty + output), `rowGroups(List)` (default all, zero-based, unique IDs in caller-specified + order), and `batches(...)` returning a Matrix-owned closeable iterator of Matrix values, + normally one batch per selected row group. Limit counts matching rows after filtering and + applies across batches. Footer IDs in range remain valid when statistics, dictionary or bloom + pruning excludes them: contribute no rows and skip the empty batch. Only out-of-range or + duplicate IDs are rejected. Preserve caller-selected order and test valid pruned IDs. Close + resources on exhaustion, explicit close and failure; document try-with-resources/withCloseable. + Iterator state stays on one input and does not prefetch unbounded groups. One row group can + exceed heap, so also support `maxBatchRows` (positive, default null) to split it; document that + byte-array cells/one huge row still constrain memory. Skip empty filtered batches and retain + schema for empty materialized results. Test early stop and cleanup. + +- 3.10 [x] Measure 3.2/3.3 against 1.6 before removing the old path. Initial acceptance target: at + least 20% lower median full-scan elapsed time on wide text and 25% lower peak live heap on + categorical data, with no greater than 5% median regression on mixed/nested data. Record + repeated-run variance; revise targets explicitly if evidence shows they are unrealistic. Timing + thresholds belong in benchmark reports, not flaky CI assertions. + +### 4. Add selective HTTP access (P2, depends on 3) + +- 4.1 [x] Implement a seekable HTTP `InputFile` / `SeekableInputStream` with bounded range + caching. Discover stable length and verify actual `206`/`Content-Range` behavior; fetch + footer/index/data ranges on demand. Reuse the same remote-object identity for metadata and rows, + detecting changes via validators where available. + +- 4.2 [x] Preserve ordinary URL reads through fallback to the existing temporary-file path when a + server lacks range support. Proposed `remoteReadMode` enum/string default `DOWNLOAD` preserves + current behavior; `AUTO` attempts ranges then downloads safely, and `RANGE` requires verified + range support. Add positive `connectTimeoutMillis` and `readTimeoutMillis`, proposed defaults + 10000/60000, plus positive `rangeCacheBytes`, proposed default 8388608. Propagate these through + builder/options/SPI. + +- 4.3 [x] Test redirects typical of Hub resolve URLs, servers ignoring Range, bad range responses, + EOF, retries with bounded attempts, resource cleanup and object changes. Do not forward + authorization across unrelated redirect origins. Keep credentials out of logs. No new Hub + authentication or Xet-client integration is required for public URLs. + +- 4.4 [x] Use a local range-capable HTTP server and selective projection/predicate fixture to + prove fewer transferred bytes than a full download. Count footer/index overhead too. Document + that this is HTTP range access, not Xet deduplication or a streaming Matrix. + +### 5. Confirm writing behavior; defer native CDC to a separate decision (P1/P3) + +- 5.1 [x] Based on 1.4, document existing page-index output. If an eligible-column omission is + reproducible, fix shared writer configuration across file/memory/stream paths; extract common + configuration rather than copying more builder chains. Add page-size tuning through the + explicitly planned options in 5.4; benchmarks inform guidance, not whether the options are + implemented. + +- 5.2 [x] Keep native CDC writing deferred unless a supported parquet-java release exposes it or a + separately scoped upstream implementation is accepted. No private upstream patch, mandatory + Python bridge, or misleading “optimized” switch in this enhancement. Page-size/row-group tuning + alone is not CDC. + +- 5.3 [x] For users needing CDC output now, document the external PyArrow rewrite below, verify + values and custom metadata survive, and state the extra I/O and Python requirement. A future + native CDC proposal must cover per-column logical hashing, nested levels, dictionary effects, + deterministic output and measured reuse after inserts/deletes. + +- 5.4 [x] Expose the safe tuning subset below through typed options, builder, SPI and shared + writer configuration. Use a Matrix-owned writer-version enum/string. Validate positive + sizes/NDVs, primitive column paths and names against the resolved schema; reject unknown map + keys/columns and invalid versions. Keep current defaults explicitly. + + | Option | Type | Proposed default | Meaning | + |---|---|---|---| + | `rowGroupSize` | long | 134217728 | Target row-group bytes | + | `pageSize` | int | 1048576 | Target encoded data-page bytes | + | `dictionaryEncoding` | boolean | true | Enable dictionary encoding with upstream fallback | + | `writerVersion` | Matrix enum/string | V1 | Select V1/V2 writer encodings and page layout | + | `columnIndexTruncateLength` | int | 64 | Binary bounds truncation for page indexes | + | `statisticsTruncateLength` | int | 2147483647 | Binary bounds truncation for row-group statistics | + | `bloomFilterEnabled` | Map | empty | Per-column overrides, globally disabled | + | `bloomFilterNDV` | Map | empty | Per-column expected distinct values; omission retains upstream sizing | + | `sizeStatisticsEnabled` | boolean | true | Collect upstream size statistics | + + V2 selects a different encoder factory as well as page layout; do not describe this as just a + page-header toggle. Keep V1 default and document that older consumers may not support all V2 + encodings; name specific consumers only after compatibility testing. Describe target-size versus + hard-limit semantics, NDV memory tradeoffs and disabled-column interactions. Test physical + metadata/page versions and logical equivalence across every output path. Add indexed equality + tests proving bloom-filter pruning independently from page/dictionary/row-group-statistics + pruning. Match any changed defaults to versioned upstream constants before implementation. + +- 5.5 [x] Track native CDC in P3: investigate [parquet-java + #3818](https://github.com/apache/parquet-java/pull/3818), referenced by + [Arrow #51684](https://github.com/apache/arrow/issues/51684) as a Java CDC implementation. The + Java PR remains open and unmerged as verified on 2026-10-17; 1.18.1 exposes no supported + CDC builder API. Recheck before choosing a future dependency. Link the applicable issue/PR and supported release here, + or draft a concrete upstream issue covering hooks/page boundaries, nested hashing and + interoperability. Ask for authorization before posting externally; a reviewable issue draft is + sufficient until then. Do not silently leave deferral untracked. + +### 6. Documentation, measurements and completion gates + +- 6.1 [x] Update module README, `docs/cookbook/matrix-parquet.md`, + `docs/tutorial/11-matrix-parquet.md` and release notes. Distinguish tested full-file + compatibility, selective local reading, selective HTTP reading and CDC writing. Include complete + examples, every new parameter/default, supported types and fallback behavior. Do not advertise + arbitrary ordinal random access from a filter API; `withFileRange` is a byte range, not a row + interval. Ordinal seeking is a separate feature. + +- 6.2 [x] Record a reproducible benchmark for full scan, narrow projection and selective + predicates on ordinary/indexed/CDC+indexed files: rows, file/page/row-group sizes, compression, + returned rows, bytes read/transferred, wall time and peak memory. Keep timing benchmarks outside + normal correctness tests; compare full-scan results to 1.6 and the measured targets in 3.10. + +- 6.3 [x] For each implementation phase run, in order: `./gradlew :matrix-parquet:codenarcMain`, + `./gradlew :matrix-parquet:spotlessCheck`, `./gradlew :matrix-parquet:test`. Run `./gradlew + test` before completing implementation. Record successful commands and outcomes here or in the + PR before checking tasks off. Include `codenarcTest` for changed tests. + +### 7. Sharded datasets and bounded parallel decoding (P2) + +- 7.1 [x] Add `read(List)`, directory input and explicit `readGlob(String)` using caller + order for lists and sorted normalized-path order for directories/globs; reject duplicate + normalized paths. No implicit glob interpretation in ordinary file strings. Directory reads + select direct `.parquet` children by default, not recursive trees. Reject empty selections with + a clear error. Check all schemas before collecting results: strict logical types, field + names/order, decimal precision/scale and timestamp units/UTC flags by default. Do not silently + coerce incompatible shards. Test Matrix metadata policy separately from physical schema + compatibility. + +- 7.2 [x] Add `sourceFileColumn` (String, default null; reject collisions) containing each shard's + normalized absolute local path. Apply projection/predicates independently per file schema and a + single matching-row limit across the dataset. Define dataset row-group selectors as explicit + per-file selections rather than ambiguous global group numbers. Extend closeable batches to + shards while preserving deterministic file/group order. + +- 7.3 [x] Add `parallelism` (positive int, default 1) and bounded prefetch (default one work item + per worker). Decode independent files/groups on separate readers; never share a seekable stream, + converter or mutable dictionary. Reuse each file's already-parsed footer through + `ParquetFileReader.open(InputFile, ParquetMetadata, ParquetReadOptions, SeekableInputStream)` + with a separate stream per worker. Verify the source identity/length matches the footer snapshot + and close streams on constructor/open failures. Test one footer parse per file rather than per + worker; do not share mutable pruning state between workers. Preserve order, cap queued results, + cancel remaining work on limit/close/failure and propagate errors with source context. Measure + throughput/heap against sequential decoding and test resource cleanup and deterministic results; + avoid decoding whole datasets before returning the first batch. + +### 8. Optional Hub URL resolution (P3) + +- 8.1 [x] Specify and test `hf://datasets//@/` resolution to an HTTPS + `/datasets///resolve//` URL; revision defaults to `main`. Match + huggingface_hub parsing/quoting, including percent-encoded revision slashes + (`refs%2Fconvert%2Fparquet`) and supported special unescaped refs. Pin compatibility to + huggingface_hub **v2.2.0** `parse_hf_uri` and URL quoting, after comparing v1.0.0 `resolve_path`, + `unquote`, special-ref regex and `safe_revision`. The new convert-ref regex allows dots/dashes; + deep-path @ is literal. Record differences and regenerate offline differential reference cases + with the adopted tagged client. Preserve unescaped refs/convert/parquet and refs/pr/N; percent-encode + other slash-containing revisions according to the selected client. Add differential URL tests + for @, nested paths, encoded and special refs, Unicode and avoiding double decoding/encoding. + [Client implementation](https://github.com/huggingface/huggingface_hub/blob/v2.2.0/src/huggingface_hub/hf_file_system.py). + Reuse phase 4 transport without adding Xet claims. Optional HF_TOKEN is sent only to approved + `https://huggingface.co` origins, never signed CDN redirect origins or logs; test redirect + behavior with mocked origins. Verify current Hub URL/auth documentation before implementation. + Existing public HTTPS reads need no token. + +Execution order: **P0: 0–2; P1: 3 and 5.1/5.4; P2: 7 and 4; P3: 8 and 5.5**. +Documentation and completion checks in 6 apply to every delivery. Phase 5.2/5.3 explains +CDC deferral and the external rewrite. Native CDC writing is not a prerequisite for reading +optimized downloads correctly. The execution record below describes the delivered implementation. + +## Usage examples and fixture reproduction + +API for a downloaded optimized file (compatibility verified by phase 1): + +```groovy +import se.alipsa.matrix.parquet.MatrixParquetReader + +def data = MatrixParquetReader.read(new File('downloaded-optimized.parquet')) +``` + +Phase 3 API; values convert through the file schema: + +```groovy +import se.alipsa.matrix.parquet.Predicates +import se.alipsa.matrix.parquet.MatrixParquetReader + +def subset = MatrixParquetReader.builder() + .columns(['id', 'text']) + .filter(Predicates.ge('id', 1000L)) + .limit(100) + .useColumnIndexFilter(true) + .read(new File('downloaded-optimized.parquet')) +``` + +Additional P1/P2 usage: + +```groovy +import java.time.LocalDate +import se.alipsa.matrix.parquet.MatrixParquetReader +import se.alipsa.matrix.parquet.Predicates +import se.alipsa.matrix.parquet.MatrixParquetWriter + +// Logical DATE comparisons require no epoch-day conversion in application code. +def preview = MatrixParquetReader.builder() + .filter(Predicates.ge('date', LocalDate.of(2026, 1, 1))) + .limit(20) + .read(new File('events.parquet')) + +// Close explicitly even if consumption stops early; limit counts matches globally. +MatrixParquetReader.builder().limit(1000).maxBatchRows(250) + .batches(new File('events.parquet')).withCloseable { batches -> + while (batches.hasNext()) { + def batch = batches.next() + // Process this Matrix before requesting the next bounded batch. + } + } + +MatrixParquetWriter.builder(preview) + .rowGroupSize(16L * 1024 * 1024) + .pageSize(64 * 1024) + .dictionaryEncoding(true) + .writerVersion('V2') + .write(new File('preview.parquet')) + +// Shard order follows this supplied list; directory/glob APIs sort paths. +def shards = MatrixParquetReader.builder().sourceFileColumn('_source').parallelism(2) + .read([new File('train-00000.parquet'), new File('train-00001.parquet')]) +``` + +For explicit lists, preserve caller order (and reject duplicate normalized paths); for +folder/glob selections use sorted normalized paths. Carry this distinction into 7.1. + +Minimal end-to-end external rewrite for CDC output. Use a dedicated environment and pin +the generator version in the fixture manifest; 21.0.0 is a suitable minimum baseline: + +```bash +python3 -m venv /tmp/matrix-parquet-fixtures +/tmp/matrix-parquet-fixtures/bin/python -m pip install pyarrow==21.0.0 +/tmp/matrix-parquet-fixtures/bin/python - <<'PY' +import pyarrow.parquet as pq + +# input.parquet is an existing matrix-parquet output file. +table = pq.read_table('input.parquet') +pq.write_table( + table, 'optimized.parquet', compression='snappy', + use_content_defined_chunking=True, write_page_index=True, +) +assert pq.read_table('optimized.parquet').equals(table, check_metadata=True) +PY +``` + +Here `table` is the full Arrow table, the second argument is the output path, +`compression` explicitly chooses SNAPPY (also PyArrow's default), CDC defaults to false +unless enabled, and page-index writing defaults to false unless enabled. Other settings +use the pinned PyArrow version's defaults. This example loads the table into memory; +the eventual fixture generator must explicitly set row-group/page/encoding options and +emit all four CDC/index combinations plus its manifest. It must not require Hub uploads. + + +## Implementation and verification record — 2026-10-17 + +All numbered deliverables above are implemented and verified in this branch. Native CDC writing +is intentionally deferred as specified by 5.2; its upstream tracking and external rewrite are +complete. The module remains on parquet-java 1.18.1, Hadoop 3.5.0 and JDK 21. + +### Final ordered gates + +These successful commands verify the delivered tree, including each separate 2.0a–2.0e task: + +```bash +./gradlew :matrix-parquet:codenarcMain :matrix-parquet:codenarcTest --console=plain +./gradlew :matrix-parquet:spotlessCheck --console=plain +PARQUET_PYTHON=/tmp/matrix-parquet-fixtures/bin/python \ + ./gradlew :matrix-parquet:test :matrix-parquet:compileConsumerApi -PrunSlowTests=true --console=plain +./gradlew test -Pheadless=true --console=plain +``` + +CodeNarc and Spotless passed. The Parquet suite reported **122 tests, 117 passed, zero failures, +five skipped**. The skips are opt-in benchmarks and original-writer fixture capture; default-size +CDC was generated and tested. The Java consumer compiles without parquet-java/Hadoop on its +compile classpath. The full repository regression task passed; unaffected tests may be up-to-date. + +| Task | Implementation and independently asserted coverage | +|---|---| +| 0.1–1.5 | 0.8.0 migration table; original 0.7.x strings/temporal/decimal fixtures; eight Arrow layout cases and manifest; physical indexes/pages; all primary entry points | +| 2.0 | Recursive STRING annotations, version marker, automatic legacy nested text compatibility and binaryAsString options | +| 2.0a | Raw binary writes and exact reads through bytes/files/streams/SPI; fixed metadata allowlist; unknown/incompatible metadata fallback; explicit legacy `[B` rejection | +| 2.0b | Byte/Short/LocalTime/Instant/UUID/Character/Enum policies; precision/overflow checks; contextual unsupported-type rejection | +| 2.0c | Shared ColumnTypePolicy descriptor; complete scans, abstract/interface inference, concrete assignability, all-null rules and incompatible Date subclass rejection in either order | +| 2.0d | Exact inferred/declared/nested BigDecimal DECIMAL routing; per-field/global precedence, disabled-inference errors, high precision, negative scale and all-null cases | +| 2.0e | Allowlist coverage for all emitted canonical types; every supported policy through bytes, streams and SPI files; validation before output mutation and migration notes | +| 2.1–2.5 | External raw media bytes, integer decimals, naive/adjusted timestamps at every unit, UUID and explicit FLOAT16/INT96 errors, legacy LIST variants | +| 3.1–3.9 | InputFile/ParquetFileReader core, column converters, accept/skip rollback, dictionary reuse, logical predicates, projection/index remapping, physical pruning, limits and closeable bounded batches | +| 4.1–4.4 | Range input, fallback, cache/timeouts, bounded redirects/retries, origin-scoped tokens, malformed/truncated ranges, object changes and fewer transferred bytes | +| 5.1–5.5 | Physical output indexes across file/memory/stream paths, shared tuning, bloom-only pruning, metadata-preserving Arrow CDC rewrite and upstream CDC tracking | +| 6.1, 6.3 | Complete README/defaults/examples, tutorial/cookbook/release updates; ordered gates and full repository suite | +| 1.6, 3.10, 6.2, 7.3 | Original wide baseline, million-row Group/columnar comparisons, sampled live heap, three Arrow layouts with full/projection/filter modes and sequential/parallel dataset measurements | +| 7.1–7.3 | Ordered list/directory/glob shards, strict schemas/metadata, source column, per-file groups, reused footers, bounded independent workers, limit and early-close cleanup | +| 8.1 | Tagged huggingface_hub 2.2.0 offline differential cases after v1.0.0 comparison; revision/path quoting and redirect credential tests | + +### Reproduction and additional successful checks + +```bash +/tmp/matrix-parquet-fixtures/bin/python matrix-parquet/src/test/python/generate_optimized.py +/tmp/matrix-hf-fixtures/bin/python matrix-parquet/src/test/python/generate_hub_urls.py +/tmp/matrix-parquet-fixtures/bin/python matrix-parquet/src/test/python/verify_matrix_output.py +./gradlew :matrix-parquet:test --tests ParquetInteropTest --console=plain +./gradlew :matrix-parquet:codecWithoutHadoop --console=plain +groovy scripts/release-all/test-workflow.groovy . +``` + +Fixture regeneration is byte-for-byte deterministic on the pinned environments. Checked-in +Parquet binaries total **737,019 bytes**, within the 4 MiB budget and 1 MiB individual cap. +Python environments use pyarrow==21.0.0 and huggingface_hub==2.2.0 respectively. Python verification +confirmed SNAPPY/ZSTD annotated strings, binary payloads and exact decimals; CDC rewrites retain +values and Matrix metadata and are reread by Java. CI runs these checks in +.github/workflows/release-all-tools.yml. Fixture provenance and client-version differences are +recorded in src/test/resources/optimized/README.md. + +Original-writer capture was run successfully with: + +```bash +./gradlew :matrix-parquet:test --tests ParquetLegacyCaptureTest \ + -I /tmp/parquet-legacy.init.gradle --no-configuration-cache --console=plain +``` + +That init script sets test system property parquet.captureLegacy=true. The capture compiles the +writer/value helper at b2095159664b34ce426685c387adf0bccdb33e01 together in an isolated loader. +The byte[] hypothesis was corrected by this end-to-end experiment: this pinned writer throws +InvalidSchemaException and cannot produce a valid byte[] fixture. The checked-in `[B` text footer +is explicitly synthetic; no original bytes are claimed recoverable. Date/Timestamp corruption is +preserved in an actual original-writer fixture and new mixed policies are rejected before output. + +The codec-exclusion spike showed that even UNCOMPRESSED writer initialization loads Hadoop's +ZlibCompressor.CompressionLevel. Local filesystem/configuration setup is removed from the new +read/write core, but runtime Hadoop dependencies remain until upstream codec linkage permits +removal. No JDK target increase or dependency upgrade is claimed. + +### Measurements + +See [the benchmark report](../benchmarks/optimized-parquet.md) for hardware/JVM/heap/GC, input +generation, warmup/variance, allocations, sampled live-heap definitions, exact byte counts and +reproduction commands. The original wide baseline was captured before production edits; the +three-workload differential comparison retains the original Group loop with shared corrected +schema conversions. Inputs are generated outside Git, and no claim of an OS-cold cache is made. + +The million-row comparison measured 89.5% lower wide median time, 43.8% lower categorical sampled +live heap and 88.2% lower mixed/nested time, meeting the initial targets with the documented +sampling method. Identical Arrow layouts exercise full scans, ID projection and 100-row selection; +page indexes reduced selective reads from 1,279,020 to 552,869 bytes (542,383 with CDC). An isolated +page-index test reads 2,914 versus 82,700 bytes with all other pruning disabled. Independent bloom +pruning reads 16,737 versus 40,373 bytes. HTTP ranges transfer 88,476 bytes of an approximately 906,000-byte object, +including footer overhead. Four-worker shard decoding measured a 242 ms warm median versus 341 ms +sequentially. These are local observations, not timing assertions or general speed guarantees. + +Benchmark commands (init-file setup is fully documented in the report): + +```bash +./gradlew :matrix-parquet:test --tests '*ParquetReadBenchmarkTest' \ + -I /tmp/parquet-baseline.init.gradle --no-configuration-cache --console=plain +./gradlew :matrix-parquet:test --tests '*ParquetDatasetBenchmarkTest' \ + -I /tmp/parquet-dataset.init.gradle --no-configuration-cache --console=plain +/tmp/matrix-parquet-fixtures/bin/python matrix-parquet/src/test/python/generate_benchmarks.py +./gradlew :matrix-parquet:test --tests '*ParquetLayoutBenchmarkTest' \ + -I /tmp/parquet-layout.init.gradle --no-configuration-cache --console=plain +``` diff --git a/matrix-parquet/src/consumerTest/java/ParquetConsumer.java b/matrix-parquet/src/consumerTest/java/ParquetConsumer.java new file mode 100644 index 000000000..23e71f202 --- /dev/null +++ b/matrix-parquet/src/consumerTest/java/ParquetConsumer.java @@ -0,0 +1,20 @@ +import java.io.File; +import java.util.List; +import se.alipsa.matrix.core.Matrix; +import se.alipsa.matrix.parquet.MatrixParquetReader; +import se.alipsa.matrix.parquet.MatrixParquetWriter; +import se.alipsa.matrix.parquet.Predicates; + +/** Compile-only check: consumers need no parquet-java or Hadoop on their compile classpath. */ +public final class ParquetConsumer { + /** Selects an ID preview without upstream predicate types. */ + public static Matrix select(File file) { + return MatrixParquetReader.builder().columns(List.of("id")) + .filter(Predicates.ge("id", 1000L)).limit(10).read(file); + } + + /** Writes using Matrix-owned tuning methods. */ + public static void save(Matrix matrix, File file) { + MatrixParquetWriter.builder(matrix).pageSize(4096).writerVersion("V2").write(file); + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnMaterializer.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnMaterializer.groovy new file mode 100644 index 000000000..95eabf08a --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnMaterializer.groovy @@ -0,0 +1,175 @@ +package se.alipsa.matrix.parquet + +import groovy.transform.PackageScope + +import org.apache.parquet.column.Dictionary +import org.apache.parquet.io.api.Binary +import org.apache.parquet.io.api.Converter +import org.apache.parquet.io.api.GroupConverter +import org.apache.parquet.io.api.PrimitiveConverter +import org.apache.parquet.io.api.RecordMaterializer +import org.apache.parquet.schema.GroupType +import org.apache.parquet.schema.LogicalTypeAnnotation +import org.apache.parquet.schema.MessageType +import org.apache.parquet.schema.PrimitiveType +import org.apache.parquet.schema.Type + +/** Column-oriented materialization; record acceptance and rollback remain in upstream hooks. */ +@PackageScope +final class ColumnMaterializer extends RecordMaterializer { + + final List columns + final Map current = [:] + private final GroupConverter root + private final int[] sizes + private final ParquetPredicate residual + + ColumnMaterializer(MessageType schema, List types, boolean binaryAsString, ParquetPredicate residual) { + columns = schema.fields.collect { [] } + sizes = new int[columns.size()] + this.residual = residual + Converter[] children = new Converter[schema.fieldCount] + schema.fields.eachWithIndex { Type field, int index -> + Closure logicalSink = residual != null && residual.columns().contains(field.name) ? { Object value -> current[field.name] = value } : null + children[index] = converter(field, types[index], binaryAsString, + { Object value -> columns[index].add(value) }, logicalSink) + } + root = new GroupConverter() { + @Override + Converter getConverter(int index) { children[index] } + + @Override + void start() { + current.clear() + columns.eachWithIndex { List column, int index -> sizes[index] = column.size() } + } + + @Override + void end() { + columns.eachWithIndex { List column, int index -> + if (column.size() == sizes[index]) { column.add(null) } + } + } + } + } + + @Override + GroupConverter getRootConverter() { root } + + @Override + Boolean getCurrentRecord() { + residual == null || residual.test(current) ? Boolean.TRUE : null + } + + @Override + void skipCurrentRecord() { + columns.eachWithIndex { List column, int index -> + if (column.size() > sizes[index]) { column.subList(sizes[index], column.size()).clear() } + } + } + + private static Converter converter(Type type, Class expected, boolean binaryAsString, Closure sink, Closure logicalSink = null) { + if (type.primitive) { + return new ValueConverter(type.asPrimitiveType(), expected, binaryAsString, sink, logicalSink) + } + GroupType group = type.asGroupType() + Map values = [:] + Converter[] children = new Converter[group.fieldCount] + group.fields.eachWithIndex { Type child, int index -> + children[index] = converter(child, null, binaryAsString) { Object value -> + if (child.repetition == Type.Repetition.REPEATED) { + ((List) values.computeIfAbsent(child.name) { [] }).add(value) + } else { + values[child.name] = value + } + } + } + new GroupConverter() { + @Override + Converter getConverter(int index) { children[index] } + + @Override + void start() { values.clear() } + + @Override + void end() { + def logical = group.logicalTypeAnnotation + if (logical instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) { + Type repeated = group.getType(0) + List entries = (List) (values[repeated.name] ?: []) + boolean wrapped = !repeated.primitive && repeated.asGroupType().fieldCount == 1 && + !(repeated.name in ['array', "${group.name}_tuple".toString()]) + sink.call(wrapped ? entries.collect { entry -> ((Map) entry)[repeated.asGroupType().getType(0).name] } : entries) + } else if (logical instanceof LogicalTypeAnnotation.MapLogicalTypeAnnotation) { + Map result = [:] + ((List) (values[group.getType(0).name] ?: [])).each { entry -> Map pair = (Map) entry; result[pair.key] = pair.value } + sink.call(result) + } else { + Map result = [:] + group.fields.each { Type child -> result[child.name] = values[child.name] } + sink.call(result) + } + } + } + } + + private static class ValueConverter extends PrimitiveConverter { + private final PrimitiveType field + private final Class expected + private final boolean binaryAsString + private final Closure sink + private final Closure logicalSink + private Object[] dictionary + private Object[] logicalDictionary + + ValueConverter(PrimitiveType field, Class expected, boolean binaryAsString, Closure sink, Closure logicalSink = null) { + this.field = field + this.expected = expected + this.binaryAsString = binaryAsString + this.sink = sink + this.logicalSink = logicalSink + } + + private void accept(Object value) { + sink.call(MatrixParquetReader.decodePrimitive(value, field, expected, binaryAsString)) + if (logicalSink != null) { logicalSink.call(logicalValue(value)) } + } + + private Object logicalValue(Object value) { + Class logicalExpected = field.logicalTypeAnnotation instanceof LogicalTypeAnnotation.TimeLogicalTypeAnnotation ? java.time.LocalTime : + field.logicalTypeAnnotation instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation && + ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) field.logicalTypeAnnotation).isAdjustedToUTC() ? java.time.Instant : + field.logicalTypeAnnotation == null && field.primitiveTypeName == PrimitiveType.PrimitiveTypeName.BINARY ? byte[] : null + MatrixParquetReader.decodePrimitive(value, field, logicalExpected, false) + } + + @Override + void addInt(int value) { accept(value) } + @Override + void addLong(long value) { accept(value) } + @Override + void addFloat(float value) { accept(value) } + @Override + void addDouble(double value) { accept(value) } + @Override + void addBoolean(boolean value) { accept(value) } + @Override + void addBinary(Binary value) { accept(value) } + @Override + boolean hasDictionarySupport() { + field.logicalTypeAnnotation instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation || expected == String + } + @Override + void setDictionary(Dictionary source) { + dictionary = new Object[source.maxId + 1] + if (logicalSink != null) { logicalDictionary = new Object[source.maxId + 1] } + (0..source.maxId).each { int id -> + Binary value = source.decodeToBinary(id) + dictionary[id] = MatrixParquetReader.decodePrimitive(value, field, expected, binaryAsString) + if (logicalSink != null) { logicalDictionary[id] = logicalValue(value) } + } + } + @Override + void addValueFromDictionary(int id) { sink.call(dictionary[id]); logicalSink?.call(logicalDictionary[id]) } + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnTypePolicy.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnTypePolicy.groovy new file mode 100644 index 000000000..a3549f13c --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnTypePolicy.groovy @@ -0,0 +1,105 @@ +package se.alipsa.matrix.parquet + +import groovy.transform.PackageScope + +import java.lang.reflect.Modifier +import java.sql.Time +import java.sql.Timestamp +import java.time.Instant +import java.time.LocalDate +import java.time.LocalDateTime +import java.time.LocalTime + +/** Validated encoding and metadata types shared by schema construction and metadata emission. */ +@PackageScope +final class ColumnTypePolicy { + + private static final int NANOS_PER_MICRO = 1000 + static final List SUPPORTED = [String, Integer, Long, Float, Double, Boolean, Byte, Short, + BigInteger, BigDecimal, Character, byte[], LocalDate, LocalDateTime, LocalTime, Instant, + UUID, Date, java.sql.Date, Time, Timestamp, List, Map] + static final Map METADATA_TYPES = SUPPORTED.collectEntries { Class type -> [(type.name): type] } + + final Class encoding + final Class metadata + + private ColumnTypePolicy(Class encoding, Class metadata) { + this.encoding = encoding + this.metadata = metadata + } + + static ColumnTypePolicy resolve(String name, List values, Class declaration) { + Class declared = box(declaration ?: Object) + boolean infer = declared == Object || declared.interface || Modifier.isAbstract(declared.modifiers) + Class policy = null + values.eachWithIndex { value, int row -> + if (value == null) { + return + } + if (!declared.isInstance(value)) { + throw new IllegalArgumentException("Column '$name', row $row: declared ${declared.name}, got ${value.getClass().name}") + } + if (value instanceof Instant) { + if (value.nano % NANOS_PER_MICRO != 0) { throw new IllegalArgumentException("Column '$name', row $row: Instant loses sub-microsecond precision") } + try { Math.addExact(Math.multiplyExact(value.epochSecond, 1_000_000L), value.nano.intdiv(NANOS_PER_MICRO)) } + catch (ArithmeticException failure) { throw new IllegalArgumentException("Column '$name', row $row: Instant overflows epoch micros", failure) } + } + Class current = policyType(value.getClass()) + if (current == null || (policy != null && policy != current)) { + throw new IllegalArgumentException("Column '$name', row $row: unsupported/mixed ${value.getClass().name}; expected ${policy?.name ?: 'supported encoding'}") + } + policy = current + } + if (policy == null) { + if (infer && !(declared in [List, Map])) { + if (!declared.isAssignableFrom(String)) { + throw new IllegalArgumentException("All-null column '$name' declared ${declared.name} requires explicit retyping") + } + policy = String + } else { + policy = policyType(declared) + } + } + if (policy == null) { + throw new IllegalArgumentException("Unsupported type ${declared.name} for column '$name'; convert explicitly") + } + Class metadata = infer || policy in [String, List, Map] ? policy : declared + if (!METADATA_TYPES.containsKey(metadata.name)) { + throw new IllegalArgumentException("Unsupported declared type ${declared.name} for column '$name'") + } + new ColumnTypePolicy(policy, metadata) + } + + static Class policyType(Class type) { + if (type == Character || Enum.isAssignableFrom(type) || CharSequence.isAssignableFrom(type)) { + return String + } + if (Collection.isAssignableFrom(type)) { + return List + } + if (Map.isAssignableFrom(type)) { + return Map + } + SUPPORTED.contains(type) ? type : null + } + + static Class box(Class type) { + if (!type.primitive) { + return type + } + [ (int): Integer, (long): Long, (float): Float, (double): Double, + (boolean): Boolean, (byte): Byte, (short): Short, (char): Character ][type] + } + + static int[] decimal(List values) { + int scale = 0 + int left = 1 + values.each { value -> + if (value instanceof BigDecimal) { + scale = Math.max(scale, value.scale()) + left = Math.max(left, value.precision() - value.scale()) + } + } + [left + scale, scale] as int[] + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DatasetBatches.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DatasetBatches.groovy new file mode 100644 index 000000000..1d81f1df7 --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DatasetBatches.groovy @@ -0,0 +1,176 @@ +package se.alipsa.matrix.parquet + +import org.apache.parquet.conf.PlainParquetConfiguration +import org.apache.parquet.hadoop.ParquetFileReader +import org.apache.parquet.hadoop.metadata.ParquetMetadata +import org.apache.parquet.io.LocalInputFile + +import se.alipsa.matrix.core.Matrix + +import java.util.concurrent.Callable +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.Future + +/** Ordered shard batches with strict schema checks and at most one prefetched batch per worker. */ +final class DatasetBatches implements MatrixBatchIterator { + private final List files + private final List footers = [] + private final List> work = [] + private final List lengths = [] + private final List modified = [] + private final ParquetReadOptions options + private final ExecutorService executor + private final Map cursors = [:] + private final Map> pending = [:] + private int current + private int submitted + private long remaining + private volatile boolean closed + private Matrix template + private Matrix ready + + /** Checks every footer before reading any rows; supplied shard order is retained. */ + DatasetBatches(List selection, ParquetReadOptions options) { + options.validate() + this.options = options + if (selection == null) { throw new IllegalArgumentException('Dataset files cannot be null') } + files = selection*.canonicalFile + if (files.empty || files.toSet().size() != files.size()) { + throw new IllegalArgumentException('Dataset files must be nonempty and unique') + } + readFooters() + validateSchemas() + selectWork() + remaining = options.limit == null ? Long.MAX_VALUE : options.limit + ParquetReadOptions emptyOptions = ParquetReadOptions.fromMap(options.toMap()) + emptyOptions.rowGroups = [] + new ParquetBatches(new LocalInputFile(files[0].toPath()), emptyOptions, files[0].name, footers[0]).withCloseable { cursor -> template = cursor.emptyMatrix() } + if (options.sourceFileColumn != null) { template.addColumn(options.sourceFileColumn, String, []) } + executor = Executors.newFixedThreadPool(Math.min(options.parallelism, Math.max(1, work.size()))) + try { + if (remaining > 0) { (0.. + if (!file.isFile()) { throw new IllegalArgumentException("Not a dataset file: $file") } + lengths.add(file.length()) + modified.add(file.lastModified()) + def input = new LocalInputFile(file.toPath()) + def readOptions = org.apache.parquet.ParquetReadOptions.builder(new PlainParquetConfiguration()).build() + input.newStream().withCloseable { stream -> footers.add(ParquetFileReader.readFooter(input, readOptions, stream)) } + } + } + + private void validateSchemas() { + def schema = footers[0].fileMetaData.schema + footers.eachWithIndex { ParquetMetadata footer, int index -> + if (footer.fileMetaData.keyValueMetaData[MatrixParquetReader.METADATA_COLUMN_TYPES] != footers[0].fileMetaData.keyValueMetaData[MatrixParquetReader.METADATA_COLUMN_TYPES]) { throw new IllegalArgumentException("Incompatible shard Matrix types: ${files[index]}") } + if (footer.fileMetaData.schema.fields != schema.fields) { throw new IllegalArgumentException("Incompatible shard schema: ${files[index]}") } + if (options.sourceFileColumn != null && schema.containsField(options.sourceFileColumn)) { throw new IllegalArgumentException('sourceFileColumn collides with dataset schema') } + } + if (options.rowGroups != null) { throw new IllegalArgumentException('Dataset rowGroups require per-file reads') } + } + + private void selectWork() { + Set paths = files*.canonicalPath.toSet() + if (!paths.containsAll(options.rowGroupsByFile.keySet())) { throw new IllegalArgumentException('Unknown shard path in rowGroupsByFile') } + files.eachWithIndex { File file, int index -> + List ids = options.rowGroupsByFile.containsKey(file.canonicalPath) ? options.rowGroupsByFile[file.canonicalPath] : (0..= footers[index].blocks.size() }) { + throw new IllegalArgumentException("Invalid/duplicate row-group IDs for $file") + } + ids.each { Integer group -> work.add([file: index, group: group]) } + } + } + + @Override + Matrix emptyMatrix() { + Matrix.builder(template.matrixName).columnNames(template.columnNames()).types(template.types()).build() + } + + private void scheduleWork() { + if (submitted >= work.size()) { return } + int index = submitted++ + int source = work[index].file + if (files[source].length() != lengths[source] || files[source].lastModified() != modified[source]) { + throw new IOException("Shard changed after footer validation: ${files[source]}") + } + ParquetReadOptions perFile = ParquetReadOptions.fromMap(options.toMap()) + perFile.limit = null + perFile.rowGroups = [work[index].group] + cursors[index] = new ParquetBatches(new LocalInputFile(files[source].toPath()), perFile, files[source].name, footers[source]) + scheduleBatch(index) + } + + private void scheduleBatch(int index) { + ParquetBatches cursor = cursors[index] + pending[index] = executor.submit({ -> + synchronized (cursor) { + closed ? null : (cursor.hasNext() ? cursor.next() : null) + } + } as Callable) + } + + @Override + boolean hasNext() { + if (ready != null) { return true } + if (closed) { return false } + try { + while (remaining > 0 && current < work.size()) { + Matrix batch = pending.remove(current).get() + if (batch == null) { + cursors.remove(current).close() + current++ + scheduleWork() + continue + } + if (batch.rowCount() > remaining) { + List truncated = [] + batch.columnNames().each { truncated.add(new ArrayList(batch.column(it).subList(0, (int) remaining))) } + batch = Matrix.builder(batch.matrixName).columnNames(batch.columnNames()).types(batch.types()).columns(truncated).build() + } + remaining -= batch.rowCount() + if (options.sourceFileColumn != null) { batch.addColumn(options.sourceFileColumn, String, [files[work[current].file].absolutePath] * batch.rowCount()) } + ready = batch + if (remaining > 0) { scheduleBatch(current) } + return true + } + close() + false + } catch (Exception failure) { + String source = current < work.size() ? files[work[current].file].absolutePath : 'dataset' + try { close() } catch (Exception cleanup) { failure.addSuppressed(cleanup) } + throw new IOException("Dataset decoding failed for $source", failure) + } + } + + @Override + Matrix next() { + if (!hasNext()) { throw new NoSuchElementException() } + Matrix result = ready + ready = null + result + } + + @Override + void close() { + if (closed) { return } + closed = true + pending.values().each { it.cancel(true) } + executor.shutdownNow() + try { + executor.awaitTermination(10, java.util.concurrent.TimeUnit.SECONDS) + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt() + } finally { + // A cancelled Future may still be leaving its decoder. Serialize close with its work. + cursors.values().each { ParquetBatches cursor -> + synchronized (cursor) { cursor.close() } + } + ready = null + } + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DecimalSchemaPolicy.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DecimalSchemaPolicy.groovy new file mode 100644 index 000000000..1708ddd83 --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DecimalSchemaPolicy.groovy @@ -0,0 +1,37 @@ +package se.alipsa.matrix.parquet + +import groovy.transform.PackageScope + +/** Shared exact-decimal settings for top-level and nested logical field paths. */ +@PackageScope +final class DecimalSchemaPolicy { + final Map settings + final boolean infer + final Integer precision + final Integer scale + final String path + final Set used + + DecimalSchemaPolicy(Map settings, boolean infer = true, Integer precision = null, + Integer scale = null, String path = '', Set used = new HashSet<>()) { + this.settings = settings + this.infer = infer + this.precision = precision + this.scale = scale + this.path = path + this.used = used + } + + DecimalSchemaPolicy child(String name) { + new DecimalSchemaPolicy(settings, infer, precision, scale, path.empty ? name : "$path.$name".toString(), used) + } + + int[] resolve(List values, int[] supplied) { + int[] explicit = settings[path] ?: supplied + if (settings.containsKey(path)) { used.add(path) } + if (explicit != null) { return explicit } + if (precision != null) { return [precision, scale] as int[] } + if (!infer) { throw new IllegalArgumentException("Decimal field '$path' requires explicit precision/scale when inference is disabled") } + ColumnTypePolicy.decimal(values) + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpRangeInputFile.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpRangeInputFile.groovy new file mode 100644 index 000000000..e43455a4a --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpRangeInputFile.groovy @@ -0,0 +1,141 @@ +package se.alipsa.matrix.parquet + +import org.apache.parquet.io.InputFile +import org.apache.parquet.io.SeekableInputStream + +import java.nio.ByteBuffer + +/** HTTP range input with a bounded per-stream cache and stable object validators. */ +final class HttpRangeInputFile implements InputFile { + private static final String CONTENT_RANGE = 'Content-Range' + private static final String ETAG = 'ETag' + private static final String LAST_MODIFIED = 'Last-Modified' + + private static final int EOF = -1 + private final URL source + private final ParquetReadOptions options + private final int cacheBytes + private final long length + private final String etag + private final String modified + + /** Verifies byte-range support immediately; errors allow AUTO callers to choose download. */ + HttpRangeInputFile(URL source, ParquetReadOptions options) { + if (!(source.protocol in ['http', 'https'])) { throw new IOException('Range reads require HTTP(S)') } + this.source = source + this.options = options + cacheBytes = Math.min(options.rangeCacheBytes, 65536) + + def response = request(0L, 0L) + def connection = response.connection + try { + String range = connection.getHeaderField(CONTENT_RANGE) + def match = range =~ /bytes 0-0\/(\d+)/ + if (!match.matches()) { throw new IOException("Invalid Content-Range: $range") } + length = Long.parseLong(match.group(1)) + if (length <= 0) { throw new IOException('Empty remote object') } + etag = connection.getHeaderField(ETAG) + modified = connection.getHeaderField(LAST_MODIFIED) + connection.inputStream.withCloseable { stream -> + if (stream.read() < 0 || stream.read() != EOF) { throw new IOException('Invalid range payload length') } + } + } finally { connection.disconnect() } + } + + private Map request(long start, long end) { + Map headers = [Range: "bytes=$start-$end".toString()] + if (etag != null) { headers['If-Match'] = etag } + else if (modified != null) { headers['If-Unmodified-Since'] = modified } + HttpURLConnection connection = HttpTransport.open(source, options, headers) + if (connection.responseCode != 206 || !(connection.getHeaderField('Content-Encoding') in [null, 'identity'])) { + int status = connection.responseCode + connection.disconnect() + throw new IOException("Range request rejected: HTTP $status") + } + [connection: connection] + } + + private byte[] fetch(long start, int count) { + long end = start + count - 1 + HttpURLConnection connection = request(start, end).connection + try { + if (connection.getHeaderField(CONTENT_RANGE) != "bytes $start-$end/$length".toString() || + (etag != null && connection.getHeaderField(ETAG) != etag) || + (modified != null && connection.getHeaderField(LAST_MODIFIED) != modified)) { + throw new IOException('Remote object/range changed during read') + } + connection.inputStream.withCloseable { stream -> + byte[] bytes = stream.readNBytes(count) + if (bytes.length != count || stream.read() != EOF) { throw new EOFException('Truncated/oversized HTTP range') } + bytes + } + } finally { connection.disconnect() } + } + + @Override + long getLength() { length } + + @Override + SeekableInputStream newStream() { + new SeekableInputStream() { + private long position + private long cacheStart = -1L + private byte[] cache = new byte[0] + private boolean closed + + @Override + long getPos() { position } + @Override + void seek(long value) { + if (value < 0 || value > length) { throw new EOFException("Invalid seek $value") } + position = value + } + @Override + int read() { + byte[] single = new byte[1] + read(single, 0, 1) < 0 ? EOF : single[0] & 0xff + } + @Override + int read(byte[] target, int offset, int count) { + Objects.checkFromIndexSize(offset, count, target.length) + if (closed) { throw new IOException('Stream closed') } + if (count == 0) { return 0 } + if (position >= length) { return EOF } + if (position < cacheStart || position >= cacheStart + cache.length) { + cacheStart = position + cache = fetch(position, (int) Math.min(cacheBytes, length - position)) + } + int copied = (int) Math.min(count, cacheStart + cache.length - position) + System.arraycopy(cache, (int) (position - cacheStart), target, offset, copied) + position += copied + copied + } + @Override + void readFully(byte[] target) { readFully(target, 0, target.length) } + @Override + void readFully(byte[] target, int offset, int count) { + int total = 0 + while (total < count) { + int result = read(target, offset + total, count - total) + if (result < 0) { throw new EOFException() } + total += result + } + } + @Override + int read(ByteBuffer target) { + byte[] bytes = new byte[Math.min(target.remaining(), cacheBytes)] + int result = read(bytes, 0, bytes.length) + if (result > 0) { target.put(bytes, 0, result) } + result + } + @Override + void readFully(ByteBuffer target) { + while (target.hasRemaining()) { + if (read(target) < 0) { throw new EOFException() } + } + } + @Override + void close() { closed = true; cache = new byte[0] } + } + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpTransport.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpTransport.groovy new file mode 100644 index 000000000..1b6afb8c8 --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpTransport.groovy @@ -0,0 +1,60 @@ +package se.alipsa.matrix.parquet + +import groovy.transform.PackageScope + +/** Shared bounded redirects, timeouts and origin-scoped Hub credentials. */ +@PackageScope +final class HttpTransport { + private static final List PROTOCOLS = ['http', 'https'] + private static final String HTTPS = 'https' + private static final List HTTPS_PORTS = [-1, 443] + + private HttpTransport() {} + + static HttpURLConnection open(URL source, ParquetReadOptions options, Map headers) { + URL target = source + int redirects = 0 + int transientFailures = 0 + while (true) { + if (!PROTOCOLS.contains(target.protocol)) { throw new IOException('HTTP(S) URL required') } + HttpURLConnection connection = (HttpURLConnection) target.openConnection() + connection.instanceFollowRedirects = false + connection.connectTimeout = options.connectTimeoutMillis + connection.readTimeout = options.readTimeoutMillis + connection.setRequestProperty('Accept-Encoding', 'identity') + headers.each { String key, String value -> connection.setRequestProperty(key, value) } + if (options.hubToken != null && target.protocol == HTTPS && target.host == 'huggingface.co' && HTTPS_PORTS.contains(target.port)) { + connection.setRequestProperty('Authorization', "Bearer ${options.hubToken}") + } + int status + try { status = connection.responseCode } catch (IOException failure) { + connection.disconnect() + if (++transientFailures >= 2) { throw failure } + continue + } + transientFailures = 0 + if (!(status in [301, 302, 303, 307, 308])) { return connection } + String location = connection.getHeaderField('Location') + connection.disconnect() + if (location == null || redirects++ >= 5) { throw new IOException('Invalid/excessive redirects') } + URL next = new URL(target, location) + if (target.protocol == HTTPS && next.protocol != HTTPS) { throw new IOException('Unsafe HTTP redirect') } + target = next + } + } + + static InputStream download(URL source, ParquetReadOptions options) { + HttpURLConnection connection = open(source, options, [:]) + int status = connection.responseCode + if (status != 200) { + connection.disconnect() + throw new IOException("Download rejected: HTTP $status") + } + InputStream stream + try { stream = connection.inputStream } catch (IOException failure) { connection.disconnect(); throw failure } + new FilterInputStream(stream) { + @Override + void close() { try { super.close() } finally { connection.disconnect() } } + } + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HubUrl.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HubUrl.groovy new file mode 100644 index 000000000..7b300c546 --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HubUrl.groovy @@ -0,0 +1,61 @@ +package se.alipsa.matrix.parquet + +import java.nio.charset.StandardCharsets +import java.util.regex.Pattern + +/** Resolves namespaced dataset hf:// URIs, following huggingface_hub v2.2.0 parsing. */ +final class HubUrl { + private static final String SLASH = '/' + private static final String ERR_FILE_PATH = 'A dataset file path is required' + private static final String PLUS = '+' + + private static final int PARTS = 3 + private static final String PREFIX = 'hf://datasets/' + private static final Pattern SPECIAL = Pattern.compile('^refs/(?:convert/[\\w.-]+|pr/\\d+)') + + private HubUrl() {} + + /** Resolves an hf://datasets/owner/repo[@revision]/path URI without contacting the Hub. */ + static URL resolve(String uri) { + if (uri == null || !uri.startsWith(PREFIX)) { throw new IllegalArgumentException('Expected hf://datasets/owner/repo[@revision]/path') } + String location = uri.substring(PREFIX.length()).replaceAll('^/+|/+$', '') + int at = location.indexOf('@') + String repo + String revision = 'main' + String path + if (at < 0 || location.substring(0, at).count(SLASH) > 1) { + String[] parts = location.split(SLASH, PARTS) + if (parts.length < PARTS) { throw new IllegalArgumentException(ERR_FILE_PATH) } + repo = parts[0] + SLASH + parts[1] + path = parts[2] + } else { + repo = location.substring(0, at) + String tail = location.substring(at + 1) + def special = SPECIAL.matcher(tail) + if (special.find()) { + revision = special.group() + path = tail.substring(revision.length()).replaceFirst('^/', '') + } else { + int slash = tail.indexOf(SLASH) + if (slash < 0) { throw new IllegalArgumentException(ERR_FILE_PATH) } + revision = tail.substring(0, slash) + path = tail.substring(slash + 1) + } + revision = URLDecoder.decode(revision.replace(PLUS, '%2B'), StandardCharsets.UTF_8) + } + if (!(repo ==~ /[^\/@]+\/[^\/@]+/) || revision.empty || path.empty) { + throw new IllegalArgumentException('Invalid dataset repository, revision or file path') + } + if (repo.contains('--') || repo.contains('..') || repo.endsWith('.git') || + repo.split(SLASH).any { !(it ==~ /[\w](?:[\w.-]*[\w])?/) } || path.startsWith(SLASH) || path.contains('//')) { + throw new IllegalArgumentException('Invalid repository id or empty path segment') + } + String encodedRevision = SPECIAL.matcher(revision).matches() ? revision : encode(revision) + String encodedPath = path.split(SLASH, -1).collect { encode(it) }.join(SLASH) + new URI("https://huggingface.co/datasets/$repo/resolve/$encodedRevision/$encodedPath").toURL() + } + + private static String encode(String value) { + URLEncoder.encode(value, StandardCharsets.UTF_8).replace(PLUS, '%20').replace('%7E', '~').replace('*', '%2A') + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/InMemoryInputFile.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/InMemoryInputFile.groovy new file mode 100644 index 000000000..5b7551a09 --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/InMemoryInputFile.groovy @@ -0,0 +1,34 @@ +package se.alipsa.matrix.parquet + +import org.apache.parquet.io.DelegatingSeekableInputStream +import org.apache.parquet.io.InputFile +import org.apache.parquet.io.SeekableInputStream + +/** Seekable Parquet input backed by a byte array, without temporary files. */ +final class InMemoryInputFile implements InputFile { + private final byte[] content + + /** Retains the supplied array; callers must not modify it during a read. */ + InMemoryInputFile(byte[] content) { + if (content == null) { throw new IllegalArgumentException('content cannot be null') } + this.content = content + } + + @Override + long getLength() { content.length } + + @Override + SeekableInputStream newStream() { + def input = new ByteArrayInputStream(content) + new DelegatingSeekableInputStream(input) { + @Override + long getPos() { content.length - input.available() } + @Override + void seek(long position) { + if (position < 0 || position > content.length) { throw new EOFException("Invalid position $position") } + input.reset() + input.skipNBytes(position) + } + } + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixBatchIterator.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixBatchIterator.groovy new file mode 100644 index 000000000..cfc9668f8 --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixBatchIterator.groovy @@ -0,0 +1,9 @@ +package se.alipsa.matrix.parquet + +import se.alipsa.matrix.core.Matrix + +/** Closeable iterator of bounded Matrix batches; close explicitly on early termination. */ +interface MatrixBatchIterator extends Iterator, Closeable { + /** Returns an empty Matrix retaining this selection's schema and types. */ + Matrix emptyMatrix() +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetReader.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetReader.groovy index a2b19e7e1..92a9251a7 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetReader.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetReader.groovy @@ -1,11 +1,8 @@ package se.alipsa.matrix.parquet -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path +import groovy.transform.PackageScope + import org.apache.parquet.example.data.Group -import org.apache.parquet.hadoop.ParquetFileReader -import org.apache.parquet.hadoop.ParquetReader -import org.apache.parquet.hadoop.example.GroupReadSupport import org.apache.parquet.schema.GroupType import org.apache.parquet.schema.LogicalTypeAnnotation import org.apache.parquet.schema.PrimitiveType @@ -15,16 +12,18 @@ import org.apache.parquet.schema.Type import se.alipsa.matrix.core.Matrix import se.alipsa.matrix.core.util.Logger +import java.nio.ByteBuffer import java.nio.file.Files import java.nio.file.StandardCopyOption import java.sql.Time +import java.sql.Timestamp import java.time.DateTimeException import java.time.Instant import java.time.LocalDate import java.time.LocalDateTime import java.time.LocalTime import java.time.ZoneId -import java.util.concurrent.ConcurrentHashMap +import java.time.ZoneOffset /** * Reads Apache Parquet files into {@link Matrix} objects. @@ -86,6 +85,9 @@ import java.util.concurrent.ConcurrentHashMap * @see Matrix */ class MatrixParquetReader { + private static final String OPEN_BRACKET = '[' + private static final String EXTENSION = '.parquet' + private static final Logger log = Logger.getLogger(MatrixParquetReader) @@ -105,7 +107,7 @@ class MatrixParquetReader { private static final ThreadLocal ZONE_ID_HOLDER = new ThreadLocal<>() /** Cache for Class.forName() results to avoid repeated reflection calls */ - private static final Map> CLASS_CACHE = new ConcurrentHashMap<>() + private static final ThreadLocal BINARY_AS_STRING = new ThreadLocal<>() private static final String ERR_PATH_NULL = 'Path cannot be null' private static final String ERR_FILE_PATH_NULL = 'File path cannot be null' @@ -152,6 +154,12 @@ class MatrixParquetReader { new ReaderBuilder() } + /** Creates a reader from a validated snapshot of typed options. */ + static ReaderBuilder builder(ParquetReadOptions options) { + if (options == null) { throw new IllegalArgumentException('options cannot be null') } + new ReaderBuilder(options) + } + /** * Fluent builder for configuring and executing Parquet read operations. * @@ -159,9 +167,75 @@ class MatrixParquetReader { */ static class ReaderBuilder { - private final ParquetReadOptions options = new ParquetReadOptions() + private final ParquetReadOptions options + + private ReaderBuilder() { this(new ParquetReadOptions()) } + + private ReaderBuilder(ParquetReadOptions value) { + options = ParquetReadOptions.fromMap(value.toMap()) + options.hubToken = value.hubToken + } + + /** Sets columns; see ParquetReadOptions for defaults and validation. */ + ReaderBuilder columns(List value) { options.columns = value; this } + + /** Sets filter; see ParquetReadOptions for defaults and validation. */ + ReaderBuilder filter(ParquetPredicate value) { options.filter = value; this } + + /** Sets binaryAsString; see ParquetReadOptions for defaults and validation. */ + ReaderBuilder binaryAsString(Boolean value) { options.binaryAsString = value; this } + + /** Sets useColumnIndexFilter; see ParquetReadOptions for defaults and validation. */ + ReaderBuilder useColumnIndexFilter(boolean value) { options.useColumnIndexFilter = value; this } + + /** Sets limit; see ParquetReadOptions for defaults and validation. */ + ReaderBuilder limit(long value) { options.limit = value; this } + + /** Sets rowGroups; see ParquetReadOptions for defaults and validation. */ + ReaderBuilder rowGroups(List value) { options.rowGroups = value; this } + + /** Sets maxBatchRows; see ParquetReadOptions for defaults and validation. */ + ReaderBuilder maxBatchRows(int value) { options.maxBatchRows = value; this } + + /** Transport: DOWNLOAD, AUTO or RANGE; default 'DOWNLOAD'. */ + ReaderBuilder remoteReadMode(String value) { options.remoteReadMode = value; this } + + /** HTTP connection timeout; default 10000. */ + ReaderBuilder connectTimeoutMillis(int value) { options.connectTimeoutMillis = value; this } + + /** HTTP read timeout; default 60000. */ + ReaderBuilder readTimeoutMillis(int value) { options.readTimeoutMillis = value; this } + + /** Maximum HTTP range cache bytes; default 8388608. */ + ReaderBuilder rangeCacheBytes(int value) { options.rangeCacheBytes = value; this } + + /** Optional local source path column; default null. */ + ReaderBuilder sourceFileColumn(String value) { options.sourceFileColumn = value; this } + + /** Bounded dataset decoder workers; default 1. */ + ReaderBuilder parallelism(int value) { options.parallelism = value; this } + + /** Reads shards in caller order after checking every schema. */ + Matrix read(List files) { + batches(files).withCloseable { iterator -> collectBatches(iterator) } + } + + /** Opens ordered shard batches; prefetch is bounded by parallelism. */ + MatrixBatchIterator batches(List files) { new DatasetBatches(files, options) } + + /** Reads explicit file globs in normalized path order. */ + Matrix readGlob(String pattern) { read(globFiles(pattern)) } + + /** Selects original footer groups per shard, keyed by canonical local file path. */ + ReaderBuilder rowGroupsByFile(Map> value) { options.rowGroupsByFile = value; this } + + /** Opens closeable batches; close explicitly when stopping before exhaustion. */ + MatrixBatchIterator batches(File file) { + if (file?.isDirectory()) { return batches(directoryFiles(file)) } + validateFile(file) + new ParquetBatches(new org.apache.parquet.io.LocalInputFile(file.toPath()), options, defaultMatrixName(file)) + } - private ReaderBuilder() {} /** * Sets the name for the resulting Matrix. @@ -203,16 +277,9 @@ class MatrixParquetReader { * @return a Matrix populated with the file contents */ Matrix read(File file) { - if (options.zoneId != null && options.matrixName != null) { - return MatrixParquetReader.read(file, options.matrixName, options.zoneId) - } - if (options.zoneId != null) { - return MatrixParquetReader.read(file, options.zoneId) - } - if (options.matrixName != null) { - return MatrixParquetReader.read(file, options.matrixName) - } - MatrixParquetReader.read(file) + if (file?.isDirectory()) { return read(directoryFiles(file)) } + validateFile(file) + readInput(new org.apache.parquet.io.LocalInputFile(file.toPath()), options, defaultMatrixName(file)) } /** @@ -235,10 +302,7 @@ class MatrixParquetReader { * @return a Matrix populated with the byte array contents */ Matrix read(byte[] content) { - if (content == null) { - throw new IllegalArgumentException(ERR_CONTENT_NULL) - } - read(new ByteArrayInputStream(content)) + readInput(new InMemoryInputFile(content), options, null) } /** @@ -248,16 +312,12 @@ class MatrixParquetReader { * @return a Matrix populated with the stream contents */ Matrix read(InputStream is) { - if (options.zoneId != null && options.matrixName != null) { - return MatrixParquetReader.read(is, options.matrixName, options.zoneId) - } - if (options.zoneId != null) { - return MatrixParquetReader.read(is, options.zoneId) - } - if (options.matrixName != null) { - return MatrixParquetReader.read(is, options.matrixName) - } - MatrixParquetReader.read(is) + if (is == null) { throw new IllegalArgumentException('InputStream cannot be null') } + File temp = File.createTempFile('matrix-parquet-', EXTENSION) + try { + Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING) + readInput(new org.apache.parquet.io.LocalInputFile(temp.toPath()), options, null) + } finally { Files.deleteIfExists(temp.toPath()) } } /** @@ -267,19 +327,19 @@ class MatrixParquetReader { * @return a Matrix populated with the URL contents */ Matrix read(URL url) { - if (options.zoneId != null && options.matrixName != null) { - return MatrixParquetReader.read(url, options.matrixName, options.zoneId) - } - if (options.zoneId != null) { - return MatrixParquetReader.read(url, options.zoneId) - } - if (options.matrixName != null) { - url.openStream().withCloseable { InputStream is -> - MatrixParquetReader.read(is, options.matrixName) + if (url == null) { throw new IllegalArgumentException(ERR_URL_NULL) } + options.validate() + if (options.remoteReadMode != 'DOWNLOAD') { + HttpRangeInputFile input + try { input = new HttpRangeInputFile(url, options) } catch (IOException failure) { + if (options.remoteReadMode == 'RANGE') { throw failure } } - } else { - MatrixParquetReader.read(url) + if (input != null) { return readInput(input, options, null) } } + if (url.protocol in ['http', 'https']) { + return HttpTransport.download(url, options).withCloseable { InputStream stream -> read(stream) } + } + url.openStream().withCloseable { InputStream stream -> read(stream) } } /** @@ -305,6 +365,10 @@ class MatrixParquetReader { if (urlString == null) { throw new IllegalArgumentException(ERR_URL_STRING_NULL) } + if (urlString.startsWith('hf://')) { + options.hubToken = System.getenv('HF_TOKEN') + return read(HubUrl.resolve(urlString)) + } read(new URI(urlString).toURL()) } } @@ -318,16 +382,6 @@ class MatrixParquetReader { zoneId != null ? zoneId : ZoneId.systemDefault() } - /** - * Gets a cached Class object for the given class name. - * Uses Thread.currentThread().contextClassLoader.loadClass() and caches the result for subsequent calls. - */ - private static Class getCachedClass(String className) { - CLASS_CACHE.computeIfAbsent(className) { name -> - Thread.currentThread().contextClassLoader.loadClass(name) - } - } - /** * Validates the file parameter before reading. * @@ -436,7 +490,7 @@ class MatrixParquetReader { if (content == null) { throw new IllegalArgumentException(ERR_CONTENT_NULL) } - read(new ByteArrayInputStream(content)) + readInput(new InMemoryInputFile(content), new ParquetReadOptions(), null) } /** @@ -453,7 +507,7 @@ class MatrixParquetReader { if (content == null) { throw new IllegalArgumentException(ERR_CONTENT_NULL) } - read(new ByteArrayInputStream(content), matrixName) + readInput(new InMemoryInputFile(content), new ParquetReadOptions().matrixName(matrixName), null) } /** @@ -470,7 +524,7 @@ class MatrixParquetReader { if (content == null) { throw new IllegalArgumentException(ERR_CONTENT_NULL) } - read(new ByteArrayInputStream(content), zoneId) + readInput(new InMemoryInputFile(content), new ParquetReadOptions().zoneId(zoneId), null) } /** @@ -488,7 +542,7 @@ class MatrixParquetReader { if (content == null) { throw new IllegalArgumentException(ERR_CONTENT_NULL) } - read(new ByteArrayInputStream(content), matrixName, zoneId) + readInput(new InMemoryInputFile(content), new ParquetReadOptions().matrixName(matrixName).zoneId(zoneId), null) } /** @@ -545,7 +599,7 @@ class MatrixParquetReader { if (urlString == null) { throw new IllegalArgumentException(ERR_URL_STRING_NULL) } - read(new URI(urlString).toURL()) + builder().readUrl(urlString) } /** @@ -759,6 +813,7 @@ class MatrixParquetReader { * @throws IllegalArgumentException if file is null, does not exist, or is a directory */ static Matrix read(File file) { + if (file?.isDirectory()) { return builder().read(file) } read(file, file == null ? null : defaultMatrixName(file), false, null) } @@ -776,58 +831,72 @@ class MatrixParquetReader { private static Matrix read(File file, String fallbackMatrixName, boolean preferSchemaName) { validateFile(file) - log.debug("Reading Parquet file: ${file.absolutePath}") - def path = new Path(file.toURI()) - def conf = new Configuration() - // Open Parquet file metadata - def footer = ParquetFileReader.readFooter(conf, path) - def keyValueMetaData = footer.getFileMetaData().getKeyValueMetaData() - def typeString = keyValueMetaData.get(METADATA_COLUMN_TYPES) - def indexString = keyValueMetaData.get(METADATA_INDEX_COLUMNS) - - List fieldTypes - if (typeString != null) { - fieldTypes = parseTypeString(typeString) - } - - ParquetReader.builder(new GroupReadSupport(), path).build().withCloseable { reader -> - String schemaName = footer.getFileMetaData().getSchema().name - String matrixName = fallbackMatrixName - if (preferSchemaName && schemaName != null && !schemaName.trim().isEmpty()) { - matrixName = schemaName - } + readInput(new org.apache.parquet.io.LocalInputFile(file.toPath()), new ParquetReadOptions(), preferSchemaName ? null : fallbackMatrixName) + } - Group row = reader.read() - GroupType schema = row != null ? row.getType() : footer.getFileMetaData().getSchema() - List fieldNames = schema.fields*.name - if (fieldTypes == null) { - fieldTypes = extractFieldTypes(schema) - } - if (row == null) { - Matrix m = Matrix.builder(matrixName) - .columnNames(fieldNames) - .types(fieldTypes) - .build() - return restoreIndex(m, indexString) - } + /** Reads a list of shards in the supplied order, with strict schema compatibility. */ + static Matrix read(List files) { builder().read(files) } + + /** Reads an explicit glob; ordinary file strings are not interpreted as patterns. */ + static Matrix readGlob(String pattern) { builder().readGlob(pattern) } + + private static List directoryFiles(File directory) { + (directory.listFiles() ?: new File[0]).findAll { it.isFile() && it.name.endsWith(EXTENSION) } + .sort { it.canonicalPath }.toList() + } + + private static List globFiles(String pattern) { + java.nio.file.Path path = java.nio.file.Path.of(pattern).toAbsolutePath().normalize() + String text = path as String + int wildcard = text.findIndexOf { it in ['*', '?', OPEN_BRACKET, '{'] } + if (wildcard < 0) { return [path.toFile()] } + int slash = text.lastIndexOf(File.separator, wildcard) + java.nio.file.Path root = java.nio.file.Path.of(text.substring(0, Math.max(1, slash))) + def matcher = java.nio.file.FileSystems.default.getPathMatcher("glob:$text") + Files.walk(root).withCloseable { paths -> + paths.filter { Files.isRegularFile(it) && matcher.matches(it) }.sorted().map { it.toFile() }.toList() + } + } + + private static Matrix collectBatches(MatrixBatchIterator iterator) { + Matrix prototype = iterator.emptyMatrix() + List columns = [] + prototype.columnNames().each { columns.add([]) } + while (iterator.hasNext()) { + Matrix batch = iterator.next() + batch.columnNames().eachWithIndex { String name, int index -> columns[index].addAll(batch.column(name)) } + } + Matrix.builder(prototype.matrixName).columnNames(prototype.columnNames()).types(prototype.types()).columns(columns).build() + } - def builder = Matrix.builder(matrixName) - .columnNames(fieldNames) - .types(fieldTypes) - - while (row != null) { - def rowData = [] - fieldNames.eachWithIndex { name, i -> - def type = fieldTypes[i] - def field = schema.getType(name) - def value = readValue(row, name, field, type) - rowData << value + @PackageScope + static T withReadZone(ZoneId zone, Closure action) { + ZoneId previous = ZONE_ID_HOLDER.get() + if (zone != null) { ZONE_ID_HOLDER.set(zone) } + try { action.call() } finally { + if (previous == null) { ZONE_ID_HOLDER.remove() } else { ZONE_ID_HOLDER.set(previous) } + } + } + + @PackageScope + static Matrix readInput(org.apache.parquet.io.InputFile input, ParquetReadOptions options, String name) { + log.debug("Reading Parquet input: $name") + ZoneId previous = ZONE_ID_HOLDER.get() + if (options.zoneId != null) { ZONE_ID_HOLDER.set(options.zoneId) } + try { + new ParquetBatches(input, options, name).withCloseable { batches -> + Matrix empty = batches.emptyMatrix() + List columns = empty.columnNames().collect { [] } + while (batches.hasNext()) { + Matrix batch = batches.next() + batch.columnNames().eachWithIndex { String column, int index -> columns[index].addAll(batch.column(column)) } } - builder.addRow(rowData as Object[]) - row = reader.read() + Matrix result = Matrix.builder(empty.matrixName).columnNames(empty.columnNames()).types(empty.types()).columns(columns).build() + if (empty.hasIndex()) { result.createIndex(empty.indexedColumns() as String[]) } + result } - Matrix m = builder.build() - restoreIndex(m, indexString) + } finally { + if (previous == null) { ZONE_ID_HOLDER.remove() } else { ZONE_ID_HOLDER.set(previous) } } } @@ -838,16 +907,18 @@ class MatrixParquetReader { * @param indexString JSON array or legacy comma-separated index column names, or null if no index * @return the matrix with index restored (or unchanged if no index metadata) */ - private static Matrix restoreIndex(Matrix matrix, String indexString) { + @PackageScope + static Matrix restoreIndex(Matrix matrix, String indexString) { if (indexString != null && !indexString.trim().isEmpty()) { - matrix.createIndex(parseIndexColumns(indexString)) + String[] indexes = parseIndexColumns(indexString) + if (indexes.every { matrix.columnNames().contains(it) }) { matrix.createIndex(indexes) } } matrix } private static String[] parseIndexColumns(String indexString) { String value = indexString.trim() - if (value.startsWith('[')) { + if (value.startsWith(OPEN_BRACKET)) { try { return parseJsonStringArray(value) as String[] } catch (IllegalArgumentException ignored) { @@ -864,7 +935,7 @@ class MatrixParquetReader { private static List parseJsonStringArray(String json) { List result = [] int pos = skipWhitespace(json, 0) - if (pos >= json.length() || json.charAt(pos) != '[' as char) { + if (pos >= json.length() || json.charAt(pos) != OPEN_BRACKET as char) { throw new IllegalArgumentException("Invalid index metadata JSON: $json") } pos = skipWhitespace(json, pos + 1) @@ -955,7 +1026,7 @@ class MatrixParquetReader { if (is == null) { throw new IllegalArgumentException('InputStream cannot be null') } - java.nio.file.Path tempFile = Files.createTempFile('matrix-parquet-', '.parquet') + java.nio.file.Path tempFile = Files.createTempFile('matrix-parquet-', EXTENSION) try { Files.copy(is, tempFile, StandardCopyOption.REPLACE_EXISTING) File file = tempFile.toFile() @@ -972,11 +1043,7 @@ class MatrixParquetReader { } } - private static List parseTypeString(String typeString) { - typeString.split(COMMA).collect { className -> - getCachedClass(className.trim()) - } - } + private static Class getJavaType(PrimitiveType.PrimitiveTypeName typeName) { switch (typeName) { @@ -985,11 +1052,13 @@ class MatrixParquetReader { case PrimitiveTypeName.FLOAT -> Float case PrimitiveTypeName.DOUBLE -> Double case PrimitiveTypeName.BOOLEAN -> Boolean + case PrimitiveTypeName.BINARY, PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY -> byte[] default -> String } } - private static List extractFieldTypes(GroupType schema) { + @PackageScope + static List extractFieldTypes(GroupType schema) { schema.fields.collect { field -> def logical = field.getLogicalTypeAnnotation() if (logical instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) { @@ -1010,6 +1079,10 @@ class MatrixParquetReader { BigDecimal } else if (logical instanceof LogicalTypeAnnotation.TimeLogicalTypeAnnotation) { Time + } else if (logical instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation) { + String + } else if (logical instanceof LogicalTypeAnnotation.UUIDLogicalTypeAnnotation) { + UUID } else { getJavaType(primitive) } @@ -1020,7 +1093,8 @@ class MatrixParquetReader { } } - private static Object readValue(Group group, String fieldName, Type fieldType, Class expectedType) { + @PackageScope + static Object readValue(Group group, String fieldName, Type fieldType, Class expectedType) { if (group.getFieldRepetitionCount(fieldName) == 0) { return null } @@ -1041,63 +1115,78 @@ class MatrixParquetReader { } private static Object readPrimitive(Group group, String fieldName, PrimitiveType fieldType, Class expectedType) { - def logical = fieldType.logicalTypeAnnotation - switch (fieldType.primitiveTypeName) { - case PrimitiveTypeName.INT32 -> { - if (logical == LogicalTypeAnnotation.dateType()) { - LocalDate.ofEpochDay(group.getInteger(fieldName, 0)) - } else if (logical instanceof LogicalTypeAnnotation.TimeLogicalTypeAnnotation) { - readTime(fieldName, group.getInteger(fieldName, 0), logical as LogicalTypeAnnotation.TimeLogicalTypeAnnotation) - } else if (expectedType == java.sql.Date) { - java.sql.Date.valueOf(LocalDate.ofEpochDay(group.getInteger(fieldName, 0))) - } else { - group.getInteger(fieldName, 0) - } - } - case PrimitiveTypeName.INT64 -> { - if (logical instanceof LogicalTypeAnnotation.TimeLogicalTypeAnnotation) { - readTime(fieldName, group.getLong(fieldName, 0), logical as LogicalTypeAnnotation.TimeLogicalTypeAnnotation) - } else if (logical instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { - Instant instant = timestampInstant(group.getLong(fieldName, 0), logical.unit) - if (expectedType == java.sql.Timestamp) { - java.sql.Timestamp.from(instant) - } else if (expectedType == Date) { - Date.from(instant) - } else { - LocalDateTime.ofInstant(instant, getZoneId()) - } - } else { - def longValue = group.getLong(fieldName, 0) - if (expectedType == BigInteger) { - BigInteger.valueOf(longValue) - } else if (expectedType == Date) { - new Date(longValue) - } else { - longValue - } - } - } + Object raw = switch (fieldType.primitiveTypeName) { + case PrimitiveTypeName.INT32 -> group.getInteger(fieldName, 0) + case PrimitiveTypeName.INT64 -> group.getLong(fieldName, 0) case PrimitiveTypeName.FLOAT -> group.getFloat(fieldName, 0) case PrimitiveTypeName.DOUBLE -> group.getDouble(fieldName, 0) case PrimitiveTypeName.BOOLEAN -> group.getBoolean(fieldName, 0) - case PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, PrimitiveTypeName.BINARY -> { - if (logical instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { - def scale = logical.scale - def binary = group.getBinary(fieldName, 0) - def unscaled = new BigInteger(binary.getBytes()) - if (expectedType == BigInteger) { - unscaled - } else { - new BigDecimal(unscaled, scale) - } - } else if (expectedType == BigDecimal) { - new BigDecimal(group.getBinary(fieldName, 0).toStringUsingUTF8()) - } else { - group.getBinary(fieldName, 0).toStringUsingUTF8() - } + case PrimitiveTypeName.BINARY, PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY -> group.getBinary(fieldName, 0) + default -> throw new IllegalArgumentException("Unsupported ${fieldType.primitiveTypeName} field $fieldName") + } + decodePrimitive(raw, fieldType, expectedType, BINARY_AS_STRING.get() == Boolean.TRUE) + } + + @PackageScope + static Object decodePrimitive(Object raw, PrimitiveType field, Class expected, boolean binaryAsString) { + if (field.primitiveTypeName == PrimitiveTypeName.INT96) { + throw new IllegalArgumentException("Unsupported INT96 field '${field.name}'; convert legacy timestamps explicitly") + } + def logical = field.logicalTypeAnnotation + if (logical instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { + BigInteger unscaled = raw instanceof org.apache.parquet.io.api.Binary ? new BigInteger(raw.bytes) : BigInteger.valueOf(((Number) raw).longValue()) + return expected == BigInteger ? unscaled : new BigDecimal(unscaled, logical.scale) + } + if (logical instanceof LogicalTypeAnnotation.DateLogicalTypeAnnotation) { + LocalDate date = LocalDate.ofEpochDay(((Number) raw).longValue()) + return expected in [java.sql.Date, Date] ? java.sql.Date.valueOf(date) : date + } + if (logical instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { + Instant instant = timestampInstant(((Number) raw).longValue(), logical.unit) + if (expected == Instant) { return instant } + if (expected == Timestamp) { return Timestamp.from(instant) } + if (expected == Date) { return Date.from(instant) } + return LocalDateTime.ofInstant(instant, logical.isAdjustedToUTC() ? getZoneId() : ZoneOffset.UTC) + } + if (logical instanceof LogicalTypeAnnotation.TimeLogicalTypeAnnotation) { + long value = ((Number) raw).longValue() + if (expected == LocalTime || (expected == null && !logical.isAdjustedToUTC())) { + long multiplier = logical.unit == LogicalTypeAnnotation.TimeUnit.MILLIS ? NANOS_PER_MILLI : + logical.unit == LogicalTypeAnnotation.TimeUnit.MICROS ? NANOS_PER_MICRO : 1L + return LocalTime.ofNanoOfDay(Math.multiplyExact(value, multiplier)) } - default -> group.getString(fieldName, 0) + return readTime(field.name, value, logical) + } + if (logical instanceof LogicalTypeAnnotation.UUIDLogicalTypeAnnotation) { + ByteBuffer buffer = ByteBuffer.wrap(((org.apache.parquet.io.api.Binary) raw).bytes) + return new UUID(buffer.getLong(), buffer.getLong()) + } + if (logical instanceof LogicalTypeAnnotation.Float16LogicalTypeAnnotation) { + throw new IllegalArgumentException("Unsupported FLOAT16 field '${field.name}'") + } + if (raw instanceof org.apache.parquet.io.api.Binary) { + if (expected == BigDecimal) { return new BigDecimal(raw.toStringUsingUTF8()) } + if (logical instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation || expected == String || + (logical == null && field.primitiveTypeName == PrimitiveTypeName.BINARY && binaryAsString && expected != byte[])) { + return raw.toStringUsingUTF8() + } + return raw.bytes + } + narrowInteger(raw, field.name, expected) + } + + private static Object narrowInteger(Object raw, String fieldName, Class expected) { + if (expected == Byte) { + long value = ((Number) raw).longValue() + if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) { throw new IllegalArgumentException("Byte field '$fieldName' is out of range") } + return ((Number) raw).byteValue() + } + if (expected == Short) { + long value = ((Number) raw).longValue() + if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { throw new IllegalArgumentException("Short field '$fieldName' is out of range") } + return ((Number) raw).shortValue() } + raw } private static Time readTime(String fieldName, long value, @@ -1129,9 +1218,7 @@ class MatrixParquetReader { time.setTime(time.time + millisecondRemainder) time } catch (DateTimeException exception) { - // Unreachable in practice: the one-day range guard above rejects out-of-range values - // before LocalTime.ofNanoOfDay can throw, so the contextual error already comes from - // the guard. Kept defensively in case a future unit change bypasses it. + // Guard defensively in case a future unit bypasses the one-day range check. throw new IllegalArgumentException("Invalid TIME value $value for field '$fieldName'", exception) } } diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy index 4e7801eb3..8f0fc5412 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy @@ -1,6 +1,5 @@ package se.alipsa.matrix.parquet -import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.parquet.example.data.Group import org.apache.parquet.example.data.simple.SimpleGroupFactory @@ -22,8 +21,10 @@ import java.beans.PropertyDescriptor import java.math.RoundingMode import java.sql.Time import java.sql.Timestamp +import java.time.Instant import java.time.LocalDate import java.time.LocalDateTime +import java.time.LocalTime import java.time.ZoneId /** @@ -116,6 +117,33 @@ class MatrixParquetWriter { static class WriterBuilder { private final Matrix matrix + /** Sets rowGroupSize; see ParquetWriteOptions for defaults. */ + WriterBuilder rowGroupSize(long value) { options.rowGroupSize(value); this } + + /** Sets pageSize; see ParquetWriteOptions for defaults. */ + WriterBuilder pageSize(int value) { options.pageSize(value); this } + + /** Sets dictionaryEncoding; see ParquetWriteOptions for defaults. */ + WriterBuilder dictionaryEncoding(boolean value) { options.dictionaryEncoding(value); this } + + /** Sets writerVersion; see ParquetWriteOptions for defaults. */ + WriterBuilder writerVersion(String value) { options.writerVersion(value); this } + + /** Sets columnIndexTruncateLength; see ParquetWriteOptions for defaults. */ + WriterBuilder columnIndexTruncateLength(int value) { options.columnIndexTruncateLength(value); this } + + /** Sets statisticsTruncateLength; see ParquetWriteOptions for defaults. */ + WriterBuilder statisticsTruncateLength(int value) { options.statisticsTruncateLength(value); this } + + /** Sets bloomFilterEnabled; see ParquetWriteOptions for defaults. */ + WriterBuilder bloomFilterEnabled(Map value) { options.bloomFilterEnabled(value); this } + + /** Sets bloomFilterNDV; see ParquetWriteOptions for defaults. */ + WriterBuilder bloomFilterNDV(Map value) { options.bloomFilterNDV(value); this } + + /** Sets sizeStatisticsEnabled; see ParquetWriteOptions for defaults. */ + WriterBuilder sizeStatisticsEnabled(boolean value) { options.sizeStatisticsEnabled(value); this } + private final ParquetWriteOptions options = new ParquetWriteOptions() private WriterBuilder(Matrix matrix) { @@ -304,7 +332,7 @@ class MatrixParquetWriter { * @param matrix the matrix to write * @param fileOrDir the target file or directory. If directory the file name will be based on the matrix name. * @param inferPrecisionAndScale whether to infer precision and scale for BigDecimal columns. If false, - * BigDecimal columns default to double storage (which leads to some loss of precision). + * BigDecimal fields require explicit precision/scale settings. * Nested Groovy {@link List} and {@link Map} structures are preserved as Parquet LIST/MAP/STRUCT types. * @return the target file * @throws IllegalArgumentException if matrix is null, has no columns, or fileOrDir is null @@ -424,10 +452,10 @@ class MatrixParquetWriter { MessageType schema = createSchema(matrix, options) if (options.zoneId != null) { return ParquetValueWriter.withZoneId(options.zoneId) { - writeInternal(matrix, file, schema, options.compressionCodec, options.roundingMode) + writeInternal(matrix, file, schema, options.compressionCodec, options.roundingMode, options) } } - return writeInternal(matrix, file, schema, options.compressionCodec, options.roundingMode) + return writeInternal(matrix, file, schema, options.compressionCodec, options.roundingMode, options) } /** @@ -435,7 +463,7 @@ class MatrixParquetWriter { * * @param matrix the matrix to write * @param inferPrecisionAndScale whether to infer precision and scale for BigDecimal columns. If false, - * BigDecimal columns default to double storage (which leads to some loss of precision). + * BigDecimal fields require explicit precision/scale settings. * @return byte array containing Parquet data * @throws IllegalArgumentException if matrix is null or has no columns */ @@ -490,10 +518,10 @@ class MatrixParquetWriter { MessageType schema = createSchema(matrix, options) if (options.zoneId != null) { return ParquetValueWriter.withZoneId(options.zoneId) { - writeBytesInternal(matrix, schema, options.compressionCodec, options.roundingMode) + writeBytesInternal(matrix, schema, options.compressionCodec, options.roundingMode, options) } } - return writeBytesInternal(matrix, schema, options.compressionCodec, options.roundingMode) + return writeBytesInternal(matrix, schema, options.compressionCodec, options.roundingMode, options) } private static File determineTargetFile(Matrix matrix, File fileOrDir) { @@ -515,26 +543,15 @@ class MatrixParquetWriter { String normalized = (matrixName ?: DEFAULT_MATRIX_NAME) .replaceAll(/[\\\/:<>"|?*\p{Cntrl}]/, '_') .replaceAll(/[. ]+$/, '') - if (normalized.trim().isEmpty() || normalized.replace('.', '').trim().isEmpty()) { + if (normalized.trim().isEmpty() || normalized.replace(DOT, '').trim().isEmpty()) { normalized = DEFAULT_MATRIX_NAME } normalized.toLowerCase(Locale.ROOT).endsWith('.parquet') ? normalized : "${normalized}.parquet" } private static MessageType createSchema(Matrix matrix, ParquetWriteOptions options) { - if (options.hasDecimalMeta()) { - return buildSchema(matrix, options.decimalMeta) - } - if (options.hasUniformPrecisionAndScale()) { - Map decimalMeta = [:] - matrix.columnNames().each { String col -> - if (matrix.type(col) == BigDecimal) { - decimalMeta[col] = [options.precision, options.scale] as int[] - } - } - return buildSchema(matrix, decimalMeta) - } - return buildSchema(matrix, options.inferPrecisionAndScale) + buildSchema(matrix, options.decimalMeta, + new DecimalSchemaPolicy(options.decimalMeta, options.inferPrecisionAndScale, options.precision, options.scale)) } /** @@ -546,22 +563,42 @@ class MatrixParquetWriter { * @return the target file */ private static File writeInternal(Matrix matrix, File file, MessageType schema, - CompressionCodecName compressionCodec = CompressionCodecName.SNAPPY, RoundingMode roundingMode = RoundingMode.UNNECESSARY) { - def conf = new Configuration() - def writer = ExampleParquetWriter.builder(new Path(file.toURI())) + CompressionCodecName compressionCodec = CompressionCodecName.SNAPPY, RoundingMode roundingMode = RoundingMode.UNNECESSARY, ParquetWriteOptions options = new ParquetWriteOptions()) { + def conf = new org.apache.parquet.conf.PlainParquetConfiguration() + def writerBuilder = ExampleParquetWriter.builder(new org.apache.parquet.io.LocalOutputFile(file.toPath())) .withConf(conf) .withWriteMode(ParquetFileWriter.Mode.OVERWRITE) .withType(schema) .withCompressionCodec(compressionCodec) .withExtraMetaData(buildExtraMeta(matrix)) - .build() + def writer = configureWriter(writerBuilder, schema, options).build() writer.withCloseable { parquetWriter -> writeRows(parquetWriter, matrix, schema, roundingMode) } return file } + private static final String FORMAT_VERSION = '2' + private static final String DOT = '.' + + private static ExampleParquetWriter.Builder configureWriter(ExampleParquetWriter.Builder builder, MessageType schema, ParquetWriteOptions options) { + options.validate() + Set paths = schema.columns.collect { it.path.join(DOT) }.toSet() + (options.bloomFilterEnabled.keySet() + options.bloomFilterNDV.keySet()).each { String path -> + if (!paths.contains(path)) { throw new IllegalArgumentException("Unknown primitive column '$path'") } + } + builder.withRowGroupSize(options.rowGroupSize).withPageSize(options.pageSize) + .withDictionaryEncoding(options.dictionaryEncoding) + .withWriterVersion(options.writerVersion == 'V1' ? org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_1_0 : org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_2_0) + .withColumnIndexTruncateLength(options.columnIndexTruncateLength) + .withStatisticsTruncateLength(options.statisticsTruncateLength) + .withSizeStatisticsEnabled(options.sizeStatisticsEnabled) + options.bloomFilterEnabled.each { String path, Boolean enabled -> builder.withBloomFilterEnabled(path, enabled) } + options.bloomFilterNDV.each { String path, Long ndv -> builder.withBloomFilterNDV(path, ndv) } + builder + } + private static Map buildExtraMeta(Matrix matrix) { - Map extraMeta = [:] - extraMeta.put(METADATA_COLUMN_TYPES, matrix.types()*.name.join(COMMA)) + Map extraMeta = ['matrix.parquet.version': FORMAT_VERSION] + extraMeta.put(METADATA_COLUMN_TYPES, matrix.columnNames().collect { String col -> ColumnTypePolicy.resolve(col, matrix.column(col), matrix.type(col)).metadata.name }.join(COMMA)) if (matrix.hasIndex()) { extraMeta.put(METADATA_INDEX_COLUMNS, toJsonStringArray(matrix.indexedColumns())) } @@ -605,15 +642,15 @@ class MatrixParquetWriter { * @return byte array containing Parquet data */ private static byte[] writeBytesInternal(Matrix matrix, MessageType schema, - CompressionCodecName compressionCodec = CompressionCodecName.SNAPPY, RoundingMode roundingMode = RoundingMode.UNNECESSARY) { - def conf = new Configuration() + CompressionCodecName compressionCodec = CompressionCodecName.SNAPPY, RoundingMode roundingMode = RoundingMode.UNNECESSARY, ParquetWriteOptions options = new ParquetWriteOptions()) { + def conf = new org.apache.parquet.conf.PlainParquetConfiguration() InMemoryOutputFile outputFile = new InMemoryOutputFile() - def writer = ExampleParquetWriter.builder(outputFile) + def writerBuilder = ExampleParquetWriter.builder(outputFile) .withConf(conf) .withType(schema) .withCompressionCodec(compressionCodec) .withExtraMetaData(buildExtraMeta(matrix)) - .build() + def writer = configureWriter(writerBuilder, schema, options).build() writer.withCloseable { parquetWriter -> writeRows(parquetWriter, matrix, schema, roundingMode) } return outputFile.getBytes() } @@ -658,15 +695,19 @@ class MatrixParquetWriter { * @param explicitDecimalMeta map of column name to [precision, scale] array for BigDecimal columns. * @return the Parquet MessageType schema. */ - private static MessageType buildSchema(Matrix matrix, Map explicitDecimalMeta) { + private static MessageType buildSchema(Matrix matrix, Map explicitDecimalMeta, DecimalSchemaPolicy decimals = null) { Map decimalMeta = withBigIntegerPrecision(matrix, explicitDecimalMeta) + DecimalSchemaPolicy policy = decimals ?: new DecimalSchemaPolicy(explicitDecimalMeta ?: [:]) def builder = Types.buildMessage() matrix.columnNames().each { col -> def type = matrix.type(col) def meta = decimalMeta?.get(col) log.debug("Building schema for column '$col' of type $type with decimal meta: $meta") - builder.addField(buildParquetType(col, matrix.column(col), type, meta)) + builder.addField(buildParquetType(col, matrix.column(col), type, meta, false, policy.child(col))) } + Set unknown = new HashSet<>(policy.settings.keySet()) + unknown.removeAll(policy.used) + if (!unknown.empty) { throw new IllegalArgumentException("Unknown/non-decimal metadata paths: $unknown") } return builder.named(matrix.matrixName ?: 'MatrixSchema') } @@ -678,7 +719,7 @@ class MatrixParquetWriter { private static Map withBigIntegerPrecision(Matrix matrix, Map explicitDecimalMeta) { Map merged = [:] + (explicitDecimalMeta ?: [:]) matrix.columnNames().each { col -> - if (matrix.type(col) != BigInteger) { + if (ColumnTypePolicy.resolve(col, matrix.column(col), matrix.type(col)).encoding != BigInteger) { return } int[] existing = merged[col] @@ -731,8 +772,8 @@ class MatrixParquetWriter { * Builds the Parquet schema based on the matrix, inferring decimal precision/scale if requested. * This is for backward compatibility and delegates to the explicit map builder. * @param matrix the matrix - * @param inferPrecisionAndScale if true, infers decimal precision and scale. If false, BigDecimal columns default to - * double storage (which leads to some loss of precision). + * @param inferPrecisionAndScale if true, infers decimal precision and scale. If false, BigDecimal fields require + * explicit precision/scale settings. * @return the Parquet MessageType schema. */ private static MessageType buildSchema(Matrix matrix, boolean inferPrecisionAndScale = false) { @@ -740,7 +781,7 @@ class MatrixParquetWriter { if (inferPrecisionAndScale) { decimalMeta = inferDecimalPrecisionAndScale(matrix) } - return buildSchema(matrix, decimalMeta) + return buildSchema(matrix, decimalMeta, new DecimalSchemaPolicy(decimalMeta ?: [:], inferPrecisionAndScale)) } /** @@ -752,35 +793,35 @@ class MatrixParquetWriter { * @param decimalMeta optional decimal precision/scale metadata for {@link BigDecimal} columns * @return a Parquet {@link Type} describing the column */ - private static Type buildParquetType(String name, List values, Class clazz, int[] decimalMeta = null, boolean preferStructForMaps = false) { - if (isListType(clazz, values)) { + private static Type buildParquetType(String name, List values, Class clazz, int[] decimalMeta = null, boolean preferStructForMaps = false, DecimalSchemaPolicy decimals = new DecimalSchemaPolicy([:])) { + ColumnTypePolicy resolved = ColumnTypePolicy.resolve(decimals.path ?: name, values, clazz) + if (resolved.encoding == List) { List nestedValues = extractListElements(values) - Type elementType = buildParquetType(FIELD_ELEMENT, nestedValues, inferClass(nestedValues, Object), null, true) + Type elementType = buildParquetType(FIELD_ELEMENT, nestedValues, Object, null, true, decimals.child(FIELD_ELEMENT)) if (elementType == null) { elementType = buildPrimitiveType(FIELD_ELEMENT, String, null, false) } return Types.optionalList().element(elementType).named(name) } - if (isMapType(clazz, values)) { + if (resolved.encoding == Map) { if (preferStructForMaps) { - return buildStructType(name, values, clazz) + return buildStructType(name, values, clazz, decimals) } - return buildMapType(name, values) + return buildMapType(name, values, decimals) } - if (isStructType(clazz, values)) { - return buildStructType(name, values, clazz) - } - - Class effectiveClass = inferClass(values, clazz) + Class effectiveClass = resolved.encoding int[] effectiveDecimalMeta = decimalMeta + if (effectiveClass == BigDecimal) { effectiveDecimalMeta = decimals.resolve(values, effectiveDecimalMeta) } + if (effectiveClass == BigInteger && decimals.settings.containsKey(decimals.path)) { decimals.used.add(decimals.path) } if (effectiveClass == BigInteger && effectiveDecimalMeta == null) { effectiveDecimalMeta = [inferBigIntegerPrecision(values), 0] as int[] } return buildPrimitiveType(name, effectiveClass, effectiveDecimalMeta, false) } + @SuppressWarnings(['AbcMetric', 'CyclomaticComplexity']) // Exhaustive physical/logical type dispatch. private static PrimitiveType buildPrimitiveType(String name, Class clazz, int[] decimalMeta = null, boolean required = false) { if (clazz == BigInteger) { if (decimalMeta == null || decimalMeta.length != DECIMAL_META_LENGTH || decimalMeta[0] <= 0 || decimalMeta[1] != 0) { @@ -804,15 +845,31 @@ class MatrixParquetWriter { .as(LogicalTypeAnnotation.decimalType(scale, precision)) return builder.named(name) } - // fallback to double (discouraged, but allowed for backward compat) - def fallbackBuilder = required ? Types.required(PrimitiveTypeName.DOUBLE) : Types.optional(PrimitiveTypeName.DOUBLE) - return fallbackBuilder.named(name) + throw new IllegalArgumentException("BigDecimal field '$name' requires precision/scale") + } + + if (clazz == UUID) { + def uuidBuilder = required ? Types.required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) : Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) + return uuidBuilder.length(16).as(LogicalTypeAnnotation.uuidType()).named(name) } def primitive def logical = null switch (clazz) { + case Byte, Short -> { + primitive = PrimitiveTypeName.INT32 + logical = LogicalTypeAnnotation.intType(clazz == Byte ? 8 : 16, true) + } + case byte[] -> primitive = PrimitiveTypeName.BINARY + case String, Character -> { + primitive = PrimitiveTypeName.BINARY + logical = LogicalTypeAnnotation.stringType() + } + case LocalTime -> { + primitive = PrimitiveTypeName.INT64 + logical = LogicalTypeAnnotation.timeType(false, LogicalTypeAnnotation.TimeUnit.NANOS) + } case Integer, int -> primitive = PrimitiveTypeName.INT32 case Long, long -> primitive = PrimitiveTypeName.INT64 case Float, float -> primitive = PrimitiveTypeName.FLOAT @@ -822,7 +879,7 @@ class MatrixParquetWriter { primitive = PrimitiveTypeName.INT32 logical = LogicalTypeAnnotation.dateType() } - case LocalDateTime, Timestamp -> { + case LocalDateTime, Timestamp, Instant -> { primitive = PrimitiveTypeName.INT64 logical = LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MICROS) } @@ -834,7 +891,7 @@ class MatrixParquetWriter { primitive = PrimitiveTypeName.INT64 logical = LogicalTypeAnnotation.timestampType(true, LogicalTypeAnnotation.TimeUnit.MILLIS) } - default -> primitive = PrimitiveTypeName.BINARY + default -> throw new IllegalArgumentException("Unsupported primitive $clazz for field $name") } def builder = required ? Types.required(primitive) : Types.optional(primitive) @@ -844,33 +901,34 @@ class MatrixParquetWriter { return builder.named(name) } - private static Type buildMapType(String name, List values) { + private static Type buildMapType(String name, List values, DecimalSchemaPolicy decimals) { def mapBuilder = Types.optionalMap() List maps = values.findAll { it instanceof Map } as List + if (maps.any { it.containsKey(null) }) { throw new IllegalArgumentException("Map field '$name' cannot contain null keys") } List keySamples = maps.collectMany { it?.keySet() ?: [] } as List Object keySample = findFirstNonNull(keySamples) Class keyClass = inferClass(keySamples, keySample?.class ?: String) - int[] keyDecimalMeta = keyClass == BigInteger ? [inferBigIntegerPrecision(keySamples), 0] as int[] : null + int[] keyDecimalMeta = keyClass == BigInteger ? [inferBigIntegerPrecision(keySamples), 0] as int[] : keyClass == BigDecimal ? decimals.child(FIELD_KEY).resolve(keySamples, null) : null PrimitiveType keyType = buildPrimitiveType(FIELD_KEY, keyClass ?: String, keyDecimalMeta, true) mapBuilder.key(keyType) List valueSamples = maps.collectMany { it?.values() ?: [] } as List Set> valueClasses = valueSamples.findAll { it != null }*.class.toSet() if (valueClasses.size() > 1) { - return buildStructType(name, values, Map) + return buildStructType(name, values, Map, decimals) } Type valueType if (valueSamples.isEmpty()) { valueType = buildPrimitiveType(FIELD_VALUE, String, null, false) } else { - valueType = buildParquetType(FIELD_VALUE, valueSamples, inferClass(valueSamples, Object), null) + valueType = buildParquetType(FIELD_VALUE, valueSamples, Object, null, false, decimals.child(FIELD_VALUE)) } mapBuilder.value(valueType) return mapBuilder.named(name) } - private static Type buildStructType(String name, List values, Class declaredType) { + private static Type buildStructType(String name, List values, Class declaredType, DecimalSchemaPolicy decimals) { Object sample = findFirstNonNull(values) if (sample == null) { return Types.optionalGroup().named(name) @@ -884,7 +942,10 @@ class MatrixParquetWriter { Set keys = new LinkedHashSet<>() values.each { val -> if (val instanceof Map mapVal) { - mapVal.keySet().each { key -> keys.add(String.valueOf(key)) } + mapVal.keySet().each { key -> + if (!(key instanceof String)) { throw new IllegalArgumentException("Struct field '$name' requires String keys; got ${key?.getClass()?.name}") } + keys.add(key) + } } } keys.each { key -> @@ -897,7 +958,7 @@ class MatrixParquetWriter { } } fieldValues[key] = collected - fieldTypes[key] = inferClass(collected, Object) + fieldTypes[key] = Object } } else { List descriptors = ParquetValueWriter.getPropertyDescriptors(sample.class) @@ -920,7 +981,7 @@ class MatrixParquetWriter { } fieldValues.each { fieldName, collected -> - Type fieldType = buildParquetType(fieldName, collected, fieldTypes[fieldName] ?: Object, null) + Type fieldType = buildParquetType(fieldName, collected, fieldTypes[fieldName] ?: Object, null, false, decimals.child(fieldName)) groupBuilder.addField(fieldType) } return groupBuilder.named(name) @@ -936,38 +997,8 @@ class MatrixParquetWriter { return elements } - private static boolean isListType(Class declaredType, List values) { - if (declaredType != null && Collection.isAssignableFrom(declaredType)) { - return true - } - return values.any { it instanceof Collection } - } - - private static boolean isMapType(Class declaredType, List values) { - if (declaredType != null && Map.isAssignableFrom(declaredType)) { - return true - } - return values.any { it instanceof Map } - } - - private static boolean isStructType(Class declaredType, List values) { - if (isMapType(declaredType, values)) { - return true - } - Class effective = inferClass(values, declaredType) - return effective != null && !isPrimitiveLike(effective) && !Collection.isAssignableFrom(effective) - } - private static Class inferClass(List values, Class declaredType) { - Object sample = findFirstNonNull(values) - if (sample != null) { - Class clazz = sample.class - return clazz.isPrimitive() ? convertPrimitive(clazz) : clazz - } - if (declaredType != null && declaredType != Object) { - return declaredType.isPrimitive() ? convertPrimitive(declaredType) : declaredType - } - return Object + ColumnTypePolicy.resolve('nested field', values, declaredType).encoding } private static Object findFirstNonNull(List values) { @@ -979,45 +1010,6 @@ class MatrixParquetWriter { return null } - private static Class convertPrimitive(Class clazz) { - if (!clazz.isPrimitive()) { - return clazz - } - switch (clazz) { - case int -> Integer - case long -> Long - case double -> Double - case float -> Float - case boolean -> Boolean - case byte -> Byte - case short -> Short - case char -> Character - default -> clazz - } - } - - private static boolean isPrimitiveLike(Class clazz) { - if (clazz == null) { - return false - } - if (clazz.isPrimitive()) { - return true - } - if (Number.isAssignableFrom(clazz)) { - return true - } - if (CharSequence.isAssignableFrom(clazz)) { - return true - } - if (Boolean.isAssignableFrom(clazz)) { - return true - } - if (Date.isAssignableFrom(clazz)) { - return true - } - return clazz in [LocalDate, LocalDateTime, Timestamp, Time] - } - /** * Infers the required maximum precision and scale for all BigDecimal columns in the matrix. * It performs the calculation in a single pass: @@ -1032,7 +1024,7 @@ class MatrixParquetWriter { Map result = [:] // columnName -> [precision, scale] matrix.columnNames().each { col -> - if (matrix.type(col) == BigDecimal) { + if (ColumnTypePolicy.resolve(col, matrix.column(col), matrix.type(col)).encoding == BigDecimal) { int maxScale = 0 int maxLeftDigits = 0 // Max digits to the left of the decimal @@ -1064,7 +1056,6 @@ class MatrixParquetWriter { return result } - private static int minBytesForPrecision(int precision) { // According to the Parquet spec: enough bits to hold [-(10^precision - 1), 10^precision - 1] // in two's complement, i.e. the bit length of the largest magnitude plus a sign bit. diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetBatches.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetBatches.groovy new file mode 100644 index 000000000..ce4f68270 --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetBatches.groovy @@ -0,0 +1,210 @@ +package se.alipsa.matrix.parquet + +import org.apache.parquet.conf.PlainParquetConfiguration +import org.apache.parquet.filter2.compat.FilterCompat +import org.apache.parquet.hadoop.ParquetFileReader +import org.apache.parquet.hadoop.metadata.ParquetMetadata +import org.apache.parquet.io.ColumnIOFactory +import org.apache.parquet.io.InputFile +import org.apache.parquet.io.RecordReader +import org.apache.parquet.schema.MessageType +import org.apache.parquet.schema.Type + +import se.alipsa.matrix.core.Matrix + +/** Closeable row-group iterator. A row group may be split with maxBatchRows; close on early exit. */ +final class ParquetBatches implements MatrixBatchIterator { + private static final String COMMA = ',' + private static final int KEEP_EMPTY = -1 + private final ParquetFileReader reader + private final ParquetReadOptions options + private final MessageType requested + private final List outputNames + private final List types + private final List selected + private final Map groupIndexes + private final ParquetPredicate residual + private final FilterCompat.Filter filter + private final String name + private final String indexMetadata + private final boolean binaryAsString + private ColumnMaterializer materializer + private RecordReader records + private org.apache.parquet.column.page.PageReadStore pages + private long rowsRemaining + private long remaining + private int groupPosition + private boolean closed + private Matrix pending + + /** Opens an input and parses its footer exactly once. */ + @SuppressWarnings('AbcMetric') // Validates and snapshots all footer-dependent state before data I/O. + ParquetBatches(InputFile input, ParquetReadOptions options, String fallbackName, ParquetMetadata snapshot = null) { + options.validate() + this.options = options + def conf = new PlainParquetConfiguration() + def baseOptions = org.apache.parquet.ParquetReadOptions.builder(conf).build() + ParquetMetadata footer = snapshot + if (footer == null) { + input.newStream().withCloseable { stream -> footer = ParquetFileReader.readFooter(input, baseOptions, stream) } + } + MessageType full = footer.fileMetaData.schema + if (full.fields.empty || full.fields.any { it.repetition == Type.Repetition.REPEATED }) { + throw new IllegalArgumentException('Parquet requires nonempty top-level fields; wrap repeated fields in LIST') + } + def metadata = footer.fileMetaData.keyValueMetaData + name = options.matrixName ?: fallbackName ?: full.name + indexMetadata = metadata[MatrixParquetReader.METADATA_INDEX_COLUMNS] + String typeString = metadata[MatrixParquetReader.METADATA_COLUMN_TYPES] + boolean legacy = typeString != null && !metadata.containsKey('matrix.parquet.version') + if (legacy && typeString.split(COMMA, KEEP_EMPTY).contains('[B')) { + throw new IllegalArgumentException('Unrecoverable legacy Matrix byte[]: original binary payload was not preserved') + } + binaryAsString = options.binaryAsString != null ? options.binaryAsString : legacy + outputNames = options.columns == null ? full.fields*.name : new ArrayList<>(options.columns) + outputNames.each { String column -> + if (!full.containsField(column)) { throw new IllegalArgumentException("Unknown column '$column'") } + } + residual = options.filter?.bind(full, options.zoneId) + Set needed = new LinkedHashSet<>(outputNames) + if (options.filter != null) { needed.addAll(options.filter.columns()) } + requested = new MessageType(full.name, full.fields.findAll { needed.contains(it.name) }) + List inferred = MatrixParquetReader.extractFieldTypes(full) + List declared = typeString == null ? [] : typeString.split(COMMA, KEEP_EMPTY).toList() + types = requested.fields.collect { Type field -> + int index = full.getFieldIndex(field.name) + Class explicit = declared.size() == full.fieldCount ? ColumnTypePolicy.METADATA_TYPES[declared[index]] : null + explicit = compatibleMetadata(explicit, field) ? explicit : null + explicit ?: (binaryAsString && field.primitive && field.asPrimitiveType().logicalTypeAnnotation == null && + field.asPrimitiveType().primitiveTypeName == org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY ? String : inferred[index]) + } + selected = options.rowGroups == null ? (0..(options.rowGroups) + selected.each { Integer id -> + if (id == null || id < 0 || id >= footer.blocks.size()) { throw new IllegalArgumentException("Invalid row-group ID $id") } + } + def pushed = residual?.pushdown(full) + filter = pushed == null ? FilterCompat.NOOP : FilterCompat.get(pushed) + def readOptions = org.apache.parquet.ParquetReadOptions.builder(conf).withRecordFilter(filter) + .useColumnIndexFilter(options.useColumnIndexFilter) + .useStatsFilter(options.useRowGroupFilters).useDictionaryFilter(options.useRowGroupFilters) + .useBloomFilter(options.useRowGroupFilters).build() + def stream = input.newStream() + try { + reader = ParquetFileReader.open(input, footer, readOptions, stream) + reader.setRequestedSchema(requested) + } catch (Exception failure) { + stream.close() + throw failure + } + groupIndexes = [:] + reader.rowGroups.eachWithIndex { block, int index -> groupIndexes[footer.blocks.indexOf(block)] = index } + remaining = options.limit == null ? Long.MAX_VALUE : options.limit + } + + private static boolean compatibleMetadata(Class expected, Type field) { + if (expected == null) { return false } + if (!field.primitive) { + boolean list = field.logicalTypeAnnotation instanceof org.apache.parquet.schema.LogicalTypeAnnotation.ListLogicalTypeAnnotation + return list ? expected == List : expected == Map + } + def primitive = field.asPrimitiveType() + def logical = primitive.logicalTypeAnnotation + if (logical instanceof org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { return expected == BigDecimal || (expected == BigInteger && logical.scale == 0) } + if (logical instanceof org.apache.parquet.schema.LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { return expected in [Date, java.sql.Timestamp, java.time.LocalDateTime, java.time.Instant] } + if (logical instanceof org.apache.parquet.schema.LogicalTypeAnnotation.TimeLogicalTypeAnnotation) { return expected in [java.sql.Time, java.time.LocalTime, Date] } + if (logical instanceof org.apache.parquet.schema.LogicalTypeAnnotation.DateLogicalTypeAnnotation) { return expected in [java.sql.Date, java.time.LocalDate, Date] } + if (logical instanceof org.apache.parquet.schema.LogicalTypeAnnotation.UUIDLogicalTypeAnnotation) { return expected == UUID } + if (logical instanceof org.apache.parquet.schema.LogicalTypeAnnotation.StringLogicalTypeAnnotation) { return expected == String } + compatiblePhysicalMetadata(expected, primitive.primitiveTypeName) + } + + private static boolean compatiblePhysicalMetadata(Class expected, + org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName physical) { + switch (physical) { + case org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32 -> expected in [Byte, Short, Integer] + case org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64 -> expected == Long + case org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT -> expected == Float + case org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE -> expected == Double + case org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BOOLEAN -> expected == Boolean + case org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY -> expected in [String, byte[], BigDecimal] + case org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY -> expected == byte[] + default -> false + } + } + + /** Returns an empty Matrix with the selected names and types. */ + Matrix emptyMatrix() { + List empty = requested.fields.collect { [] } + build(empty) + } + + private Matrix build(List data) { + List indexes = outputNames.collect { requested.getFieldIndex(it) } + List copied = indexes.collect { data[it] } + Matrix result = Matrix.builder(name).columnNames(outputNames).types(indexes.collect { types[it] }) + .columns(copied).build() + MatrixParquetReader.restoreIndex(result, indexMetadata) + } + + @Override + boolean hasNext() { + MatrixParquetReader.withReadZone(options.zoneId) { advance() } + } + + private boolean advance() { + if (pending != null) { return true } + if (closed) { return false } + try { + while (remaining > 0) { + if (rowsRemaining == 0) { + pages?.close() + pages = null + if (groupPosition >= selected.size()) { close(); return false } + Integer index = groupIndexes[selected[groupPosition++]] + if (index == null) { continue } + pages = reader.readFilteredRowGroup(index) + if (pages == null) { continue } + rowsRemaining = pages.rowCount + materializer = new ColumnMaterializer(requested, types, binaryAsString, residual) + def columnIO = new ColumnIOFactory(reader.fileMetaData.createdBy).getColumnIO(requested, reader.fileMetaData.schema, true) + records = columnIO.getRecordReader(pages, materializer, filter) + } + int cap = (int) Math.min(remaining, options.maxBatchRows ?: Integer.MAX_VALUE) + while (rowsRemaining > 0 && materializer.columns[0].size() < cap) { + if (Thread.currentThread().isInterrupted()) { throw new InterruptedIOException('Parquet decoding cancelled') } + try { records.read() } catch (Exception failure) { materializer.skipCurrentRecord(); throw failure } + rowsRemaining-- + } + if (!materializer.columns[0].empty) { + int count = materializer.columns[0].size() + pending = build(materializer.columns) + materializer.columns.each { it.clear() } + remaining -= count + return true + } + } + close() + false + } catch (Exception failure) { + close() + throw failure + } + } + + @Override + Matrix next() { + if (!hasNext()) { throw new NoSuchElementException() } + Matrix result = pending + pending = null + result + } + + @Override + void close() { + if (!closed) { + closed = true + pending = null + try { pages?.close() } finally { reader.close() } + } + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetFormatProvider.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetFormatProvider.groovy index a35d731e8..3458092c8 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetFormatProvider.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetFormatProvider.groovy @@ -33,49 +33,17 @@ class ParquetFormatProvider extends AbstractFormatProvider { @Override Matrix read(File file, Map options) { - ParquetReadOptions readOptions = ParquetReadOptions.fromMap(options) - if (readOptions.matrixName != null && readOptions.zoneId != null) { - return MatrixParquetReader.read(file, readOptions.matrixName, readOptions.zoneId) - } - if (readOptions.matrixName != null) { - return MatrixParquetReader.read(file, readOptions.matrixName) - } - if (readOptions.zoneId != null) { - return MatrixParquetReader.read(file, readOptions.zoneId) - } - MatrixParquetReader.read(file) + MatrixParquetReader.builder(ParquetReadOptions.fromMap(options)).read(file) } @Override Matrix read(URL url, Map options) { - ParquetReadOptions readOptions = ParquetReadOptions.fromMap(options) - if (readOptions.matrixName != null && readOptions.zoneId != null) { - return MatrixParquetReader.read(url, readOptions.matrixName, readOptions.zoneId) - } - if (readOptions.matrixName != null) { - url.openStream().withCloseable { InputStream is -> - MatrixParquetReader.read(is, readOptions.matrixName) - } - } else if (readOptions.zoneId != null) { - MatrixParquetReader.read(url, readOptions.zoneId) - } else { - MatrixParquetReader.read(url) - } + MatrixParquetReader.builder(ParquetReadOptions.fromMap(options)).read(url) } @Override - Matrix read(InputStream is, Map options) { - ParquetReadOptions readOptions = ParquetReadOptions.fromMap(options) - if (readOptions.matrixName != null && readOptions.zoneId != null) { - return MatrixParquetReader.read(is, readOptions.matrixName, readOptions.zoneId) - } - if (readOptions.matrixName != null) { - return MatrixParquetReader.read(is, readOptions.matrixName) - } - if (readOptions.zoneId != null) { - return MatrixParquetReader.read(is, readOptions.zoneId) - } - MatrixParquetReader.read(is) + Matrix read(InputStream stream, Map options) { + MatrixParquetReader.builder(ParquetReadOptions.fromMap(options)).read(stream) } @Override diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetPredicate.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetPredicate.groovy new file mode 100644 index 000000000..bd4f46ea5 --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetPredicate.groovy @@ -0,0 +1,210 @@ +package se.alipsa.matrix.parquet + +import groovy.transform.PackageScope + +import org.apache.parquet.filter2.predicate.FilterApi +import org.apache.parquet.filter2.predicate.FilterPredicate +import org.apache.parquet.io.api.Binary +import org.apache.parquet.schema.LogicalTypeAnnotation +import org.apache.parquet.schema.MessageType +import org.apache.parquet.schema.PrimitiveType + +import java.time.Instant +import java.time.LocalDate +import java.time.LocalDateTime +import java.time.LocalTime +import java.time.ZoneId + +/** Immutable logical predicate; values are converted against the file schema before reading. */ +final class ParquetPredicate { + private static final String IS_NOT_NULL = 'isNotNull' + + private static final String KEY_EQ = 'eq' + private static final String KEY_NE = 'ne' + private static final String KEY_LT = 'lt' + private static final String KEY_LE = 'le' + private static final String KEY_GT = 'gt' + private static final String KEY_GE = 'ge' + + final String operation + final String column + private final Object value + final List children + + @PackageScope + ParquetPredicate(String operation, String column, Object value, List children = []) { + if (column == null && (children.empty || children.any { it == null })) { throw new IllegalArgumentException('Predicate children cannot be null') } + if (column != null && (column.empty || (!(operation in ['isNull', IS_NOT_NULL]) && value == null))) { throw new IllegalArgumentException('Use explicit null predicates for null comparisons') } + this.operation = operation + this.column = column + this.value = value instanceof byte[] ? ((byte[]) value).clone() : + value instanceof Number && !(value instanceof Double) && !(value instanceof Float) ? new BigDecimal(value.toString()) : value + this.children = Collections.unmodifiableList(new ArrayList<>(children)) + } + + /** Returns the top-level columns required by this predicate. */ + Set columns() { + column == null ? children.collectMany { it.columns() }.toSet() : [column] as Set + } + + /** Evaluates the exact logical predicate; comparisons with null are false except null tests. */ + boolean test(Map row) { + switch (operation) { + case 'and' -> children.every { it.test(row) } + case 'or' -> children.any { it.test(row) } + case 'not' -> !children[0].test(row) + case 'isNull' -> row[column] == null + case IS_NOT_NULL -> row[column] != null + default -> compare(row[column]) + } + } + + private boolean compare(Object actual) { + if (actual == null || value == null) { return false } + int comparison + if (actual instanceof Number && value instanceof Number) { + if (Double.isNaN(actual.doubleValue()) || Double.isNaN(((Number) value).doubleValue())) { + return operation == KEY_NE + } + boolean actualInfinite = (actual instanceof Double || actual instanceof Float) && Double.isInfinite(actual.doubleValue()) + boolean valueInfinite = (value instanceof Double || value instanceof Float) && Double.isInfinite(((Number) value).doubleValue()) + comparison = actualInfinite && valueInfinite ? Double.compare(actual.doubleValue(), ((Number) value).doubleValue()) : + actualInfinite ? (actual.doubleValue() > 0 ? 1 : -1) : valueInfinite ? (((Number) value).doubleValue() > 0 ? -1 : 1) : + new BigDecimal(actual.toString()) <=> new BigDecimal(value.toString()) + } else if (actual instanceof String && value instanceof String) { + comparison = Arrays.compareUnsigned(actual.getBytes(java.nio.charset.StandardCharsets.UTF_8), + ((String) value).getBytes(java.nio.charset.StandardCharsets.UTF_8)) + } else if (actual instanceof byte[] && value instanceof byte[]) { + comparison = Arrays.compareUnsigned((byte[]) actual, (byte[]) value) + } else { + comparison = ((Comparable) actual) <=> value + } + switch (operation) { + case KEY_EQ -> comparison == 0 + case KEY_NE -> comparison != 0 + case KEY_LT -> comparison < 0 + case KEY_LE -> comparison <= 0 + case KEY_GT -> comparison > 0 + case KEY_GE -> comparison >= 0 + default -> throw new IllegalArgumentException("Unknown predicate $operation") + } + } + + // Only exact physical conversions are pushed down; unsafe expressions remain wholly residual. + @PackageScope + FilterPredicate pushdown(MessageType schema) { + if (column == null) { + List predicates = children*.pushdown(schema) + if (predicates.any { it == null }) { return null } + return switch (operation) { + case 'and' -> FilterApi.and(predicates[0], predicates[1]) + case 'or' -> FilterApi.or(predicates[0], predicates[1]) + default -> null + } + } + TypeCheck.validate(this, schema) + PrimitiveType field = schema.getType(column).asPrimitiveType() + def logical = field.logicalTypeAnnotation + if (operation in [KEY_NE, IS_NOT_NULL]) { return null } + if (logical instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation && + field.primitiveTypeName in [PrimitiveType.PrimitiveTypeName.BINARY, PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY]) { return null } + if (!(field.primitiveTypeName in [PrimitiveType.PrimitiveTypeName.INT32, PrimitiveType.PrimitiveTypeName.INT64, PrimitiveType.PrimitiveTypeName.BINARY])) { return null } + Object physical = physicalValue(field) + def col + switch (field.primitiveTypeName) { + case PrimitiveType.PrimitiveTypeName.INT32 -> { + col = FilterApi.intColumn(column) + if (physical != null) { physical = new BigDecimal(physical.toString()).intValueExact() } + } + case PrimitiveType.PrimitiveTypeName.INT64 -> { + col = FilterApi.longColumn(column) + if (physical != null) { physical = new BigDecimal(physical.toString()).longValueExact() } + } + case PrimitiveType.PrimitiveTypeName.BINARY -> { + col = FilterApi.binaryColumn(column) + physical = physical instanceof String ? Binary.fromString((String) physical) : physical instanceof byte[] ? Binary.fromConstantByteArray((byte[]) physical) : null + } + default -> col = null + } + buildPhysical(operation, col, (Comparable) physical) + } + + @groovy.transform.CompileDynamic + private static FilterPredicate buildPhysical(String operation, col, Comparable physical) { + switch (operation) { + case KEY_EQ, 'isNull' -> FilterApi.eq(col, physical) + case KEY_LT -> FilterApi.lt(col, physical) + case KEY_LE -> FilterApi.ltEq(col, physical) + case KEY_GT -> FilterApi.gt(col, physical) + case KEY_GE -> FilterApi.gtEq(col, physical) + default -> null + } + } + + @PackageScope + ParquetPredicate bind(MessageType schema, ZoneId zone) { + if (column == null) { return new ParquetPredicate(operation, null, null, children*.bind(schema, zone)) } + TypeCheck.validate(this, schema) + PrimitiveType field = schema.getType(column).asPrimitiveType() + Object normalized = value + if (field.logicalTypeAnnotation instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation && + ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) field.logicalTypeAnnotation).isAdjustedToUTC() && value instanceof LocalDateTime) { + normalized = value.atZone(zone ?: ZoneId.systemDefault()).toInstant() + } + ParquetPredicate bound = new ParquetPredicate(operation, column, normalized) + bound.physicalValue(field) + bound + } + + private Object physicalValue(PrimitiveType field) { + if (value == null) { return null } + def logical = field.logicalTypeAnnotation + if (logical instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) { + BigDecimal decimal = new BigDecimal(value.toString()).setScale(logical.scale, java.math.RoundingMode.UNNECESSARY) + if (decimal.precision() > logical.precision) { throw new IllegalArgumentException("Decimal predicate overflows '${field.name}'") } + return decimal.unscaledValue() + } + if (value instanceof LocalDate) { return Math.toIntExact(value.toEpochDay()) } + if (logical instanceof LogicalTypeAnnotation.TimeLogicalTypeAnnotation) { + long nanos = ((LocalTime) value).toNanoOfDay() + long divisor = logical.unit == LogicalTypeAnnotation.TimeUnit.MILLIS ? 1_000_000L : logical.unit == LogicalTypeAnnotation.TimeUnit.MICROS ? 1000L : 1L + if (nanos % divisor != 0) { throw new IllegalArgumentException("Lossy TIME predicate '${field.name}'") } + return nanos.intdiv(divisor) + } + if (logical instanceof LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) { + Instant instant = value instanceof Instant ? (Instant) value : ((LocalDateTime) value).toInstant(java.time.ZoneOffset.UTC) + long divisor = logical.unit == LogicalTypeAnnotation.TimeUnit.MILLIS ? 1_000_000L : logical.unit == LogicalTypeAnnotation.TimeUnit.MICROS ? 1000L : 1L + if (instant.nano % divisor != 0) { throw new IllegalArgumentException("Lossy TIMESTAMP predicate '${field.name}'") } + return Math.addExact(Math.multiplyExact(instant.epochSecond, 1_000_000_000L.intdiv(divisor)), instant.nano.intdiv(divisor)) + } + if (field.primitiveTypeName == PrimitiveType.PrimitiveTypeName.INT32) { return new BigDecimal(value.toString()).intValueExact() } + if (field.primitiveTypeName == PrimitiveType.PrimitiveTypeName.INT64) { return new BigDecimal(value.toString()).longValueExact() } + value + } + + private static class TypeCheck { + static void validate(ParquetPredicate predicate, MessageType schema) { + if (!schema.containsField(predicate.column)) { + throw new IllegalArgumentException("Unknown predicate column '${predicate.column}'") + } + def field = schema.getType(predicate.column) + if (!field.primitive || field.repetition == org.apache.parquet.schema.Type.Repetition.REPEATED) { + throw new IllegalArgumentException("Repeated/nested predicate '${predicate.column}' is unsupported") + } + if (predicate.operation in ['isNull', IS_NOT_NULL]) { return } + Class inferred = MatrixParquetReader.extractFieldTypes(new MessageType('predicate', [field]))[0] + Object value = predicate.value + boolean compatible = Number.isAssignableFrom(inferred) ? value instanceof Number : + inferred == java.sql.Time ? value instanceof LocalTime : + inferred == LocalDateTime ? (value instanceof LocalDateTime || + (value instanceof Instant && ((LogicalTypeAnnotation.TimestampLogicalTypeAnnotation) field.logicalTypeAnnotation).isAdjustedToUTC())) : inferred.isInstance(value) + if (!compatible) { throw new IllegalArgumentException("Incompatible predicate value for '${predicate.column}': expected $inferred") } + } + } + + @PackageScope + void validate(MessageType schema) { + if (column != null) { TypeCheck.validate(this, schema) } + children.each { it.validate(schema) } + } +} diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetReadOptions.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetReadOptions.groovy index 13e2a50d6..02c6a5e76 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetReadOptions.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetReadOptions.groovy @@ -9,6 +9,98 @@ import java.time.ZoneId * Typed options for Parquet read operations via the SPI. */ class ParquetReadOptions { + private static final String KEY_REMOTE_READ_MODE = 'remoteReadMode' + private static final String DOWNLOAD = 'DOWNLOAD' + private static final String KEY_CONNECT_TIMEOUT_MILLIS = 'connectTimeoutMillis' + private static final String KEY_READ_TIMEOUT_MILLIS = 'readTimeoutMillis' + private static final String KEY_RANGE_CACHE_BYTES = 'rangeCacheBytes' + private static final String KEY_SOURCE_FILE_COLUMN = 'sourceFileColumn' + private static final String KEY_PARALLELISM = 'parallelism' + private static final String KEY_LIMIT = 'limit' + private static final String KEY_ROW_GROUPS_BY_FILE = 'rowGroupsByFile' + private static final String KEY_ROW_GROUPS = 'rowGroups' + private static final String KEY_MAX_BATCH_ROWS = 'maxBatchRows' + + private static final String KEY_COLUMNS = 'columns' + private static final String KEY_FILTER = 'filter' + private static final String KEY_BINARY_AS_STRING = 'binaryAsString' + private static final String KEY_USE_COLUMN_INDEX_FILTER = 'useColumnIndexFilter' + + String remoteReadMode = DOWNLOAD + int connectTimeoutMillis = 10000 + int readTimeoutMillis = 60000 + int rangeCacheBytes = 8388608 + + @groovy.transform.PackageScope + String hubToken = null + String sourceFileColumn = null + int parallelism = 1 + + @groovy.transform.PackageScope + boolean useRowGroupFilters = true + + List columns = null + ParquetPredicate filter = null + Boolean binaryAsString = null + boolean useColumnIndexFilter = true + Long limit = null + List rowGroups = null + Map> rowGroupsByFile = [:] + Integer maxBatchRows = null + + /** Validates bounds and duplicate selectors before opening data pages. */ + void validate() { + if (!(remoteReadMode in [DOWNLOAD, 'AUTO', 'RANGE'])) { throw new IllegalArgumentException('remoteReadMode must be DOWNLOAD, AUTO or RANGE') } + if (connectTimeoutMillis <= 0 || readTimeoutMillis <= 0 || rangeCacheBytes <= 0) { throw new IllegalArgumentException('HTTP timeouts and rangeCacheBytes must be positive') } + + if (parallelism <= 0) { throw new IllegalArgumentException('parallelism must be positive') } + if (limit != null && limit < 0) { throw new IllegalArgumentException('limit must be nonnegative') } + if (maxBatchRows != null && maxBatchRows <= 0) { throw new IllegalArgumentException('maxBatchRows must be positive') } + if (columns != null && (columns.empty || columns.toSet().size() != columns.size())) { throw new IllegalArgumentException('columns must be nonempty and unique') } + if (rowGroups != null && rowGroups.toSet().size() != rowGroups.size()) { throw new IllegalArgumentException('rowGroups must be unique') } + } + + /** Transport: DOWNLOAD, AUTO or RANGE; default DOWNLOAD. */ + ParquetReadOptions remoteReadMode(String value) { this.remoteReadMode = value; this } + + /** HTTP connection timeout; default 10000. */ + ParquetReadOptions connectTimeoutMillis(int value) { this.connectTimeoutMillis = value; this } + + /** HTTP read timeout; default 60000. */ + ParquetReadOptions readTimeoutMillis(int value) { this.readTimeoutMillis = value; this } + + /** Maximum HTTP range cache bytes; default 8388608. */ + ParquetReadOptions rangeCacheBytes(int value) { this.rangeCacheBytes = value; this } + + /** Optional local source path column; default null. */ + ParquetReadOptions sourceFileColumn(String value) { this.sourceFileColumn = value; this } + + /** Bounded dataset decoder workers; default 1. */ + ParquetReadOptions parallelism(int value) { this.parallelism = value; this } + + /** Ordered output columns; default null. */ + ParquetReadOptions columns(List value) { this.columns = value; this } + + /** Exact logical row filter; default null. */ + ParquetReadOptions filter(ParquetPredicate value) { this.filter = value; this } + + /** Unannotated BINARY override; default null. */ + ParquetReadOptions binaryAsString(Boolean value) { this.binaryAsString = value; this } + + /** Enable page-index pruning; default true. */ + ParquetReadOptions useColumnIndexFilter(boolean value) { this.useColumnIndexFilter = value; this } + + /** Maximum accepted rows; default null. */ + ParquetReadOptions limit(Long value) { this.limit = value; this } + + /** Ordered original footer IDs; default null. */ + ParquetReadOptions rowGroups(List value) { this.rowGroups = value; this } + + /** Per-file original footer group IDs, keyed by canonical local path; default empty/all. */ + ParquetReadOptions rowGroupsByFile(Map> value) { rowGroupsByFile = value; this } + + /** Maximum rows per batch; default null. */ + ParquetReadOptions maxBatchRows(Integer value) { this.maxBatchRows = value; this } String matrixName = null ZoneId zoneId = null @@ -36,12 +128,19 @@ class ParquetReadOptions { if (zoneId != null) { result.zoneId = zoneId } + [KEY_COLUMNS, KEY_FILTER, KEY_BINARY_AS_STRING, KEY_USE_COLUMN_INDEX_FILTER, KEY_LIMIT, KEY_ROW_GROUPS, KEY_ROW_GROUPS_BY_FILE, KEY_MAX_BATCH_ROWS, KEY_REMOTE_READ_MODE, KEY_CONNECT_TIMEOUT_MILLIS, KEY_READ_TIMEOUT_MILLIS, KEY_RANGE_CACHE_BYTES, KEY_SOURCE_FILE_COLUMN, KEY_PARALLELISM].each { String key -> + result[key] = this[key] + } result } static ParquetReadOptions fromMap(Map options) { ParquetReadOptions result = new ParquetReadOptions() Map normalized = OptionMaps.normalizeKeys(options) + Set known = descriptors()*.name*.toLowerCase(Locale.ROOT).toSet() + Set unknown = new HashSet<>(normalized.keySet()) + unknown.removeAll(known) + if (!unknown.empty) { throw new IllegalArgumentException("Unknown Parquet options: $unknown") } if (normalized.containsKey('matrixname')) { String matrixName = OptionMaps.stringValueOrNull(normalized.matrixname) if (matrixName != null) { @@ -56,17 +155,51 @@ class ParquetReadOptions { result.zoneId(String.valueOf(value)) } } + applySelection(result, normalized) + result.validate() result } + private static void applySelection(ParquetReadOptions result, Map normalized) { + if (normalized.containsKey(KEY_COLUMNS)) { result.columns = (List) normalized.columns } + if (normalized.containsKey(KEY_FILTER)) { result.filter = (ParquetPredicate) normalized.filter } + if (normalized.containsKey('binaryasstring')) { result.binaryAsString = OptionMaps.booleanValueOrNull(normalized.binaryasstring, KEY_BINARY_AS_STRING) } + if (normalized.containsKey('usecolumnindexfilter')) { result.useColumnIndexFilter = OptionMaps.booleanValueOrNull(normalized.usecolumnindexfilter, KEY_USE_COLUMN_INDEX_FILTER) } + if (normalized.limit != null) { result.limit = ((Number) normalized.limit).longValue() } + if (normalized.rowgroupsbyfile != null) { result.rowGroupsByFile = (Map>) normalized.rowgroupsbyfile } + if (normalized.rowgroups != null) { result.rowGroups = (List) normalized.rowgroups } + if (normalized.maxbatchrows != null) { result.maxBatchRows = ((Number) normalized.maxbatchrows).intValue() } + if (normalized.remotereadmode != null) { result.remoteReadMode = String.valueOf(normalized.remotereadmode) } + if (normalized.connecttimeoutmillis != null) { result.connectTimeoutMillis = ((Number) normalized.connecttimeoutmillis).intValue() } + if (normalized.readtimeoutmillis != null) { result.readTimeoutMillis = ((Number) normalized.readtimeoutmillis).intValue() } + if (normalized.rangecachebytes != null) { result.rangeCacheBytes = ((Number) normalized.rangecachebytes).intValue() } + if (normalized.sourcefilecolumn != null) { result.sourceFileColumn = String.valueOf(normalized.sourcefilecolumn) } + if (normalized.parallelism != null) { result.parallelism = ((Number) normalized.parallelism).intValue() } + } + static String describe() { OptionDescriptor.describe(descriptors()) } static List descriptors() { + ParquetReadOptions defaults = new ParquetReadOptions() [ new OptionDescriptor('matrixName', String, null, 'Name for the resulting Matrix'), - new OptionDescriptor('zoneId', ZoneId, null, 'Time zone to use when reading timestamp values') + new OptionDescriptor('zoneId', ZoneId, null, 'Time zone to use when reading timestamp values'), + new OptionDescriptor(KEY_REMOTE_READ_MODE, String, defaults.remoteReadMode, 'Transport: DOWNLOAD, AUTO or RANGE'), + new OptionDescriptor(KEY_CONNECT_TIMEOUT_MILLIS, Integer, defaults.connectTimeoutMillis, 'HTTP connection timeout'), + new OptionDescriptor(KEY_READ_TIMEOUT_MILLIS, Integer, defaults.readTimeoutMillis, 'HTTP read timeout'), + new OptionDescriptor(KEY_RANGE_CACHE_BYTES, Integer, defaults.rangeCacheBytes, 'Maximum HTTP range cache bytes'), + new OptionDescriptor(KEY_SOURCE_FILE_COLUMN, String, null, 'Optional local source path column'), + new OptionDescriptor(KEY_PARALLELISM, Integer, defaults.parallelism, 'Bounded dataset decoder workers'), + new OptionDescriptor(KEY_COLUMNS, List, null, 'Ordered output columns'), + new OptionDescriptor(KEY_FILTER, ParquetPredicate, null, 'Exact logical row filter'), + new OptionDescriptor(KEY_BINARY_AS_STRING, Boolean, null, 'Unannotated BINARY override'), + new OptionDescriptor(KEY_USE_COLUMN_INDEX_FILTER, Boolean, defaults.useColumnIndexFilter, 'Enable page-index pruning'), + new OptionDescriptor(KEY_LIMIT, Long, null, 'Maximum accepted rows'), + new OptionDescriptor(KEY_ROW_GROUPS, List, null, 'Ordered original footer IDs'), + new OptionDescriptor(KEY_ROW_GROUPS_BY_FILE, Map, null, 'Per-file footer IDs keyed by canonical local path'), + new OptionDescriptor(KEY_MAX_BATCH_ROWS, Integer, null, 'Maximum rows per batch') ] } } diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetValueWriter.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetValueWriter.groovy index 5de9d92dc..6e093b1d1 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetValueWriter.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetValueWriter.groovy @@ -13,11 +13,13 @@ import org.apache.parquet.schema.Type import java.beans.Introspector import java.beans.PropertyDescriptor import java.math.RoundingMode +import java.nio.ByteBuffer import java.sql.Time import java.sql.Timestamp import java.time.Instant import java.time.LocalDate import java.time.LocalDateTime +import java.time.LocalTime import java.time.ZoneId import java.util.concurrent.ConcurrentHashMap @@ -120,7 +122,7 @@ final class ParquetValueWriter { static void writePrimitiveValue(Group group, String fieldName, PrimitiveType field, Object value, RoundingMode roundingMode) { switch (value.class) { - case Integer, int -> group.append(fieldName, (int) value) + case Byte, Short, Integer, int -> group.append(fieldName, (int) value) case Long, long -> group.append(fieldName, ((Number) value).longValue()) case BigInteger -> group.add(fieldName, Binary.fromConstantByteArray(((BigInteger) value).toByteArray())) case Float, float -> group.append(fieldName, ((Number) value).floatValue()) @@ -132,8 +134,23 @@ final class ParquetValueWriter { case Time -> group.append(fieldName, timeToMillis((Time) value)) case LocalDateTime -> group.append(fieldName, instantToMicros(((LocalDateTime) value).atZone(getZoneId()).toInstant())) case Timestamp -> group.append(fieldName, instantToMicros(((Timestamp) value).toInstant())) + case byte[] -> group.add(fieldName, Binary.fromConstantByteArray((byte[]) value)) + case LocalTime -> group.append(fieldName, ((LocalTime) value).toNanoOfDay()) + case Instant -> { + Instant instant = (Instant) value + if (instant.nano % NANOS_PER_MICRO != 0) { + throw new IllegalArgumentException("Instant field '$fieldName' loses sub-microsecond precision") + } + group.append(fieldName, instantToMicros(instant)) + } + case UUID -> { + UUID uuid = (UUID) value + group.add(fieldName, Binary.fromConstantByteArray(ByteBuffer.allocate(16).putLong(uuid.mostSignificantBits).putLong(uuid.leastSignificantBits).array())) + } + case String, Character, CharSequence -> group.append(fieldName, value.toString()) + case Enum -> group.append(fieldName, ((Enum) value).name()) case Date -> group.append(fieldName, ((Date) value).time) - default -> group.append(fieldName, value.toString()) + default -> throw new IllegalArgumentException("Unsupported ${value.getClass().name} for field $fieldName") } } @@ -143,7 +160,7 @@ final class ParquetValueWriter { } private static long instantToMicros(Instant instant) { - instant.epochSecond * MICROS_PER_SECOND + (long) (instant.nano / NANOS_PER_MICRO) + Math.addExact(Math.multiplyExact(instant.epochSecond, MICROS_PER_SECOND), (long) (instant.nano / NANOS_PER_MICRO)) } private static void writeBigDecimalValue(Group group, String fieldName, PrimitiveType field, BigDecimal value, @@ -154,7 +171,7 @@ final class ParquetValueWriter { writeFixedLenBigDecimal(group, fieldName, field, value, (LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) logical, roundingMode) } else { - group.append(fieldName, value.doubleValue()) + throw new IllegalArgumentException("Missing DECIMAL annotation for $fieldName") } } diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetWriteOptions.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetWriteOptions.groovy index 84a085dda..e1c9dec3a 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetWriteOptions.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetWriteOptions.groovy @@ -12,6 +12,12 @@ import java.time.ZoneId * Typed options for Parquet write operations via the SPI. */ class ParquetWriteOptions { + private static final String KEY_DICTIONARY_ENCODING = 'dictionaryEncoding' + private static final String KEY_WRITER_VERSION = 'writerVersion' + private static final String KEY_SIZE_STATISTICS_ENABLED = 'sizeStatisticsEnabled' + + private static final String KEY_V1 = 'V1' + private static final String KEY_PRECISION = 'precision' private static final String KEY_SCALE = 'scale' @@ -19,6 +25,51 @@ class ParquetWriteOptions { private static final String KEY_ROUNDING_MODE = 'roundingMode' private static final String ERR_ROUNDING_MODE_NULL = 'roundingMode cannot be null' + long rowGroupSize = 134217728L + + /** Sets rowGroupSize (default 134217728L); target sizes are not hard memory limits. */ + ParquetWriteOptions rowGroupSize(long value) { this.rowGroupSize = value; this } + + int pageSize = 1048576 + + /** Sets pageSize (default 1048576); target sizes are not hard memory limits. */ + ParquetWriteOptions pageSize(int value) { this.pageSize = value; this } + + boolean dictionaryEncoding = true + + /** Sets dictionaryEncoding (default true); target sizes are not hard memory limits. */ + ParquetWriteOptions dictionaryEncoding(boolean value) { this.dictionaryEncoding = value; this } + + String writerVersion = KEY_V1 + + /** Sets writerVersion (default KEY_V1); target sizes are not hard memory limits. */ + ParquetWriteOptions writerVersion(String value) { this.writerVersion = value; this } + + int columnIndexTruncateLength = 64 + + /** Sets columnIndexTruncateLength (default 64); target sizes are not hard memory limits. */ + ParquetWriteOptions columnIndexTruncateLength(int value) { this.columnIndexTruncateLength = value; this } + + int statisticsTruncateLength = Integer.MAX_VALUE + + /** Sets statisticsTruncateLength (default Integer.MAX_VALUE); target sizes are not hard memory limits. */ + ParquetWriteOptions statisticsTruncateLength(int value) { this.statisticsTruncateLength = value; this } + + Map bloomFilterEnabled = [:] + + /** Sets bloomFilterEnabled (default [:]); target sizes are not hard memory limits. */ + ParquetWriteOptions bloomFilterEnabled(Map value) { this.bloomFilterEnabled = value; this } + + Map bloomFilterNDV = [:] + + /** Sets bloomFilterNDV (default [:]); target sizes are not hard memory limits. */ + ParquetWriteOptions bloomFilterNDV(Map value) { this.bloomFilterNDV = value; this } + + boolean sizeStatisticsEnabled = true + + /** Sets sizeStatisticsEnabled (default true); target sizes are not hard memory limits. */ + ParquetWriteOptions sizeStatisticsEnabled(boolean value) { this.sizeStatisticsEnabled = value; this } + boolean inferPrecisionAndScale = true Integer precision = null Integer scale = null @@ -120,6 +171,12 @@ class ParquetWriteOptions { } void validate() { + if (rowGroupSize <= 0 || pageSize <= 0 || columnIndexTruncateLength <= 0 || statisticsTruncateLength <= 0) { + throw new IllegalArgumentException('Writer sizes/truncate lengths must be positive') + } + if (!(writerVersion in [KEY_V1, 'V2'])) { throw new IllegalArgumentException('writerVersion must be V1 or V2') } + if (bloomFilterNDV.values().any { it == null || it <= 0 }) { throw new IllegalArgumentException('Bloom NDVs must be positive') } + if ((precision == null) != (scale == null)) { throw new IllegalArgumentException('precision and scale must be provided together') } @@ -157,12 +214,18 @@ class ParquetWriteOptions { if (zoneId != null) { result.zoneId = zoneId } + result.putAll([rowGroupSize: rowGroupSize, pageSize: pageSize, dictionaryEncoding: dictionaryEncoding, writerVersion: writerVersion, columnIndexTruncateLength: columnIndexTruncateLength, statisticsTruncateLength: statisticsTruncateLength, bloomFilterEnabled: bloomFilterEnabled, bloomFilterNDV: bloomFilterNDV, sizeStatisticsEnabled: sizeStatisticsEnabled]) result } + @SuppressWarnings('AbcMetric') // Explicit typed option normalization. static ParquetWriteOptions fromMap(Map options) { ParquetWriteOptions result = new ParquetWriteOptions() Map normalized = OptionMaps.normalizeKeys(options) + Set known = descriptors()*.name*.toLowerCase(Locale.ROOT).toSet() + Set unknown = new HashSet<>(normalized.keySet()) + unknown.removeAll(known) + if (!unknown.empty) { throw new IllegalArgumentException("Unknown Parquet options: $unknown") } if (normalized.containsKey('inferprecisionandscale')) { Object inferPrecisionAndScale = normalized.inferprecisionandscale if (inferPrecisionAndScale != null) { @@ -216,6 +279,15 @@ class ParquetWriteOptions { result.roundingMode(String.valueOf(value)) } } + if (normalized.containsKey('rowgroupsize')) { result.rowGroupSize = ((Number) normalized.rowgroupsize).longValue() } + if (normalized.containsKey('pagesize')) { result.pageSize = ((Number) normalized.pagesize).intValue() } + if (normalized.containsKey('dictionaryencoding')) { result.dictionaryEncoding = OptionMaps.booleanValueOrNull(normalized.dictionaryencoding, KEY_DICTIONARY_ENCODING) } + if (normalized.containsKey('writerversion')) { result.writerVersion = String.valueOf(normalized.writerversion) } + if (normalized.containsKey('columnindextruncatelength')) { result.columnIndexTruncateLength = ((Number) normalized.columnindextruncatelength).intValue() } + if (normalized.containsKey('statisticstruncatelength')) { result.statisticsTruncateLength = ((Number) normalized.statisticstruncatelength).intValue() } + if (normalized.containsKey('bloomfilterenabled')) { result.bloomFilterEnabled = (Map) normalized.bloomfilterenabled } + if (normalized.containsKey('bloomfilterndv')) { result.bloomFilterNDV = (Map) normalized.bloomfilterndv } + if (normalized.containsKey('sizestatisticsenabled')) { result.sizeStatisticsEnabled = OptionMaps.booleanValueOrNull(normalized.sizestatisticsenabled, KEY_SIZE_STATISTICS_ENABLED) } result.validate() result } @@ -281,6 +353,7 @@ class ParquetWriteOptions { } static List descriptors() { + ParquetWriteOptions defaults = new ParquetWriteOptions() [ new OptionDescriptor(KEY_INFER_PRECISION_AND_SCALE, Boolean, Boolean.TRUE.toString(), 'Infer precision and scale for BigDecimal columns'), new OptionDescriptor(KEY_PRECISION, Integer, null, 'Uniform precision for all BigDecimal columns'), @@ -288,7 +361,16 @@ class ParquetWriteOptions { new OptionDescriptor('decimalMeta', Map, null, 'Map of column names to [precision, scale] arrays'), new OptionDescriptor('zoneId', ZoneId, null, 'Time zone to use when writing timestamp values'), new OptionDescriptor('compressionCodec', CompressionCodecName, CompressionCodecName.SNAPPY.name(), 'Compression codec for the Parquet file (e.g. SNAPPY, GZIP, ZSTD, UNCOMPRESSED)'), - new OptionDescriptor(KEY_ROUNDING_MODE, RoundingMode, RoundingMode.UNNECESSARY.name(), 'Rounding mode for explicit DECIMAL scale conversion') + new OptionDescriptor(KEY_ROUNDING_MODE, RoundingMode, RoundingMode.UNNECESSARY.name(), 'Rounding mode for explicit DECIMAL scale conversion'), + new OptionDescriptor('rowGroupSize', Long, defaults.rowGroupSize, 'Target row-group bytes'), + new OptionDescriptor('pageSize', Integer, defaults.pageSize, 'Target data-page bytes'), + new OptionDescriptor(KEY_DICTIONARY_ENCODING, Boolean, defaults.dictionaryEncoding, 'Enable dictionary encoding'), + new OptionDescriptor(KEY_WRITER_VERSION, String, defaults.writerVersion, 'V1 or V2 encodings and page layout'), + new OptionDescriptor('columnIndexTruncateLength', Integer, defaults.columnIndexTruncateLength, 'Truncated binary page bounds bytes'), + new OptionDescriptor('statisticsTruncateLength', Integer, defaults.statisticsTruncateLength, 'Truncated row-group binary statistics bytes'), + new OptionDescriptor('bloomFilterEnabled', Map, null, 'Per-column bloom-filter enabled overrides'), + new OptionDescriptor('bloomFilterNDV', Map, null, 'Per-column positive expected distinct values'), + new OptionDescriptor(KEY_SIZE_STATISTICS_ENABLED, Boolean, defaults.sizeStatisticsEnabled, 'Collect column size statistics') ] } } diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/Predicates.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/Predicates.groovy new file mode 100644 index 000000000..f4432573f --- /dev/null +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/Predicates.groovy @@ -0,0 +1,276 @@ +package se.alipsa.matrix.parquet + +import java.time.* + +/** Factory for immutable, schema-aware predicates. Null comparisons use explicit null tests. */ +final class Predicates { + private static final String KEY_EQ = 'eq' + private static final String KEY_NE = 'ne' + private static final String KEY_LT = 'lt' + private static final String KEY_LE = 'le' + private static final String KEY_GT = 'gt' + private static final String KEY_GE = 'ge' + + private Predicates() {} + + /** Compares a column using eq; values must match its logical schema. */ + static ParquetPredicate eq(String column, Number value) { + new ParquetPredicate(KEY_EQ, column, value) + } + + /** Compares a column using eq; values must match its logical schema. */ + static ParquetPredicate eq(String column, String value) { + new ParquetPredicate(KEY_EQ, column, value) + } + + /** Compares a column using eq; values must match its logical schema. */ + static ParquetPredicate eq(String column, Boolean value) { + new ParquetPredicate(KEY_EQ, column, value) + } + + /** Compares a column using eq; values must match its logical schema. */ + static ParquetPredicate eq(String column, LocalDate value) { + new ParquetPredicate(KEY_EQ, column, value) + } + + /** Compares a column using eq; values must match its logical schema. */ + static ParquetPredicate eq(String column, LocalDateTime value) { + new ParquetPredicate(KEY_EQ, column, value) + } + + /** Compares a column using eq; values must match its logical schema. */ + static ParquetPredicate eq(String column, Instant value) { + new ParquetPredicate(KEY_EQ, column, value) + } + + /** Compares a column using eq; values must match its logical schema. */ + static ParquetPredicate eq(String column, LocalTime value) { + new ParquetPredicate(KEY_EQ, column, value) + } + + /** Compares a column using eq; values must match its logical schema. */ + static ParquetPredicate eq(String column, byte[] value) { + new ParquetPredicate(KEY_EQ, column, value) + } + + /** Compares a column using ne; values must match its logical schema. */ + static ParquetPredicate ne(String column, Number value) { + new ParquetPredicate(KEY_NE, column, value) + } + + /** Compares a column using ne; values must match its logical schema. */ + static ParquetPredicate ne(String column, String value) { + new ParquetPredicate(KEY_NE, column, value) + } + + /** Compares a column using ne; values must match its logical schema. */ + static ParquetPredicate ne(String column, Boolean value) { + new ParquetPredicate(KEY_NE, column, value) + } + + /** Compares a column using ne; values must match its logical schema. */ + static ParquetPredicate ne(String column, LocalDate value) { + new ParquetPredicate(KEY_NE, column, value) + } + + /** Compares a column using ne; values must match its logical schema. */ + static ParquetPredicate ne(String column, LocalDateTime value) { + new ParquetPredicate(KEY_NE, column, value) + } + + /** Compares a column using ne; values must match its logical schema. */ + static ParquetPredicate ne(String column, Instant value) { + new ParquetPredicate(KEY_NE, column, value) + } + + /** Compares a column using ne; values must match its logical schema. */ + static ParquetPredicate ne(String column, LocalTime value) { + new ParquetPredicate(KEY_NE, column, value) + } + + /** Compares a column using ne; values must match its logical schema. */ + static ParquetPredicate ne(String column, byte[] value) { + new ParquetPredicate(KEY_NE, column, value) + } + + /** Compares a column using lt; values must match its logical schema. */ + static ParquetPredicate lt(String column, Number value) { + new ParquetPredicate(KEY_LT, column, value) + } + + /** Compares a column using lt; values must match its logical schema. */ + static ParquetPredicate lt(String column, String value) { + new ParquetPredicate(KEY_LT, column, value) + } + + /** Compares a column using lt; values must match its logical schema. */ + static ParquetPredicate lt(String column, Boolean value) { + new ParquetPredicate(KEY_LT, column, value) + } + + /** Compares a column using lt; values must match its logical schema. */ + static ParquetPredicate lt(String column, LocalDate value) { + new ParquetPredicate(KEY_LT, column, value) + } + + /** Compares a column using lt; values must match its logical schema. */ + static ParquetPredicate lt(String column, LocalDateTime value) { + new ParquetPredicate(KEY_LT, column, value) + } + + /** Compares a column using lt; values must match its logical schema. */ + static ParquetPredicate lt(String column, Instant value) { + new ParquetPredicate(KEY_LT, column, value) + } + + /** Compares a column using lt; values must match its logical schema. */ + static ParquetPredicate lt(String column, LocalTime value) { + new ParquetPredicate(KEY_LT, column, value) + } + + /** Compares a column using lt; values must match its logical schema. */ + static ParquetPredicate lt(String column, byte[] value) { + new ParquetPredicate(KEY_LT, column, value) + } + + /** Compares a column using le; values must match its logical schema. */ + static ParquetPredicate le(String column, Number value) { + new ParquetPredicate(KEY_LE, column, value) + } + + /** Compares a column using le; values must match its logical schema. */ + static ParquetPredicate le(String column, String value) { + new ParquetPredicate(KEY_LE, column, value) + } + + /** Compares a column using le; values must match its logical schema. */ + static ParquetPredicate le(String column, Boolean value) { + new ParquetPredicate(KEY_LE, column, value) + } + + /** Compares a column using le; values must match its logical schema. */ + static ParquetPredicate le(String column, LocalDate value) { + new ParquetPredicate(KEY_LE, column, value) + } + + /** Compares a column using le; values must match its logical schema. */ + static ParquetPredicate le(String column, LocalDateTime value) { + new ParquetPredicate(KEY_LE, column, value) + } + + /** Compares a column using le; values must match its logical schema. */ + static ParquetPredicate le(String column, Instant value) { + new ParquetPredicate(KEY_LE, column, value) + } + + /** Compares a column using le; values must match its logical schema. */ + static ParquetPredicate le(String column, LocalTime value) { + new ParquetPredicate(KEY_LE, column, value) + } + + /** Compares a column using le; values must match its logical schema. */ + static ParquetPredicate le(String column, byte[] value) { + new ParquetPredicate(KEY_LE, column, value) + } + + /** Compares a column using gt; values must match its logical schema. */ + static ParquetPredicate gt(String column, Number value) { + new ParquetPredicate(KEY_GT, column, value) + } + + /** Compares a column using gt; values must match its logical schema. */ + static ParquetPredicate gt(String column, String value) { + new ParquetPredicate(KEY_GT, column, value) + } + + /** Compares a column using gt; values must match its logical schema. */ + static ParquetPredicate gt(String column, Boolean value) { + new ParquetPredicate(KEY_GT, column, value) + } + + /** Compares a column using gt; values must match its logical schema. */ + static ParquetPredicate gt(String column, LocalDate value) { + new ParquetPredicate(KEY_GT, column, value) + } + + /** Compares a column using gt; values must match its logical schema. */ + static ParquetPredicate gt(String column, LocalDateTime value) { + new ParquetPredicate(KEY_GT, column, value) + } + + /** Compares a column using gt; values must match its logical schema. */ + static ParquetPredicate gt(String column, Instant value) { + new ParquetPredicate(KEY_GT, column, value) + } + + /** Compares a column using gt; values must match its logical schema. */ + static ParquetPredicate gt(String column, LocalTime value) { + new ParquetPredicate(KEY_GT, column, value) + } + + /** Compares a column using gt; values must match its logical schema. */ + static ParquetPredicate gt(String column, byte[] value) { + new ParquetPredicate(KEY_GT, column, value) + } + + /** Compares a column using ge; values must match its logical schema. */ + static ParquetPredicate ge(String column, Number value) { + new ParquetPredicate(KEY_GE, column, value) + } + + /** Compares a column using ge; values must match its logical schema. */ + static ParquetPredicate ge(String column, String value) { + new ParquetPredicate(KEY_GE, column, value) + } + + /** Compares a column using ge; values must match its logical schema. */ + static ParquetPredicate ge(String column, Boolean value) { + new ParquetPredicate(KEY_GE, column, value) + } + + /** Compares a column using ge; values must match its logical schema. */ + static ParquetPredicate ge(String column, LocalDate value) { + new ParquetPredicate(KEY_GE, column, value) + } + + /** Compares a column using ge; values must match its logical schema. */ + static ParquetPredicate ge(String column, LocalDateTime value) { + new ParquetPredicate(KEY_GE, column, value) + } + + /** Compares a column using ge; values must match its logical schema. */ + static ParquetPredicate ge(String column, Instant value) { + new ParquetPredicate(KEY_GE, column, value) + } + + /** Compares a column using ge; values must match its logical schema. */ + static ParquetPredicate ge(String column, LocalTime value) { + new ParquetPredicate(KEY_GE, column, value) + } + + /** Compares a column using ge; values must match its logical schema. */ + static ParquetPredicate ge(String column, byte[] value) { + new ParquetPredicate(KEY_GE, column, value) + } + + /** Tests whether the column is null. */ + static ParquetPredicate isNull(String column) { new ParquetPredicate('isNull', column, null) } + + /** Tests whether the column is non-null. */ + static ParquetPredicate isNotNull(String column) { new ParquetPredicate('isNotNull', column, null) } + + /** Combines two predicates with AND. */ + static ParquetPredicate and(ParquetPredicate left, ParquetPredicate right) { + new ParquetPredicate('and', null, null, [left, right]) + } + + /** Combines two predicates with OR. */ + static ParquetPredicate or(ParquetPredicate left, ParquetPredicate right) { + new ParquetPredicate('or', null, null, [left, right]) + } + + /** Negates an exact predicate, including its documented null behavior. */ + static ParquetPredicate not(ParquetPredicate predicate) { + new ParquetPredicate('not', null, null, [predicate]) + } +} diff --git a/matrix-parquet/src/test/groovy/HubUrlTest.groovy b/matrix-parquet/src/test/groovy/HubUrlTest.groovy new file mode 100644 index 000000000..fe1e1970d --- /dev/null +++ b/matrix-parquet/src/test/groovy/HubUrlTest.groovy @@ -0,0 +1,20 @@ +import static org.junit.jupiter.api.Assertions.* + +import org.junit.jupiter.api.Test + +import se.alipsa.matrix.parquet.HubUrl + +/** Dataset cases pinned to huggingface_hub v2.2.0 parse_hf_uri and file URL quoting. */ +class HubUrlTest { + @Test + void revisionsAndLiteralFileNames() { + new File('src/test/resources/optimized/hub-urls.tsv').readLines('UTF-8') + .findAll { !it.startsWith('#') }.each { String line -> + String[] parts = line.split('\t', 2) + assertEquals(parts[1], HubUrl.resolve(parts[0]).toString()) + } + ['hf://models/owner/repo/file', 'hf://datasets/owner/repo@/file', 'hf://datasets/owner/repo'].each { value -> + assertThrows(IllegalArgumentException) { HubUrl.resolve(value) } + } + } +} diff --git a/matrix-parquet/src/test/groovy/MatrixParquetTest.groovy b/matrix-parquet/src/test/groovy/MatrixParquetTest.groovy index c7caa3d19..d270cb4b7 100644 --- a/matrix-parquet/src/test/groovy/MatrixParquetTest.groovy +++ b/matrix-parquet/src/test/groovy/MatrixParquetTest.groovy @@ -114,7 +114,7 @@ class MatrixParquetTest { def empData = Matrix.builder('empData').columns( emp_id: 1..5, emp_name: ['Rick', 'Dan', 'Michelle', 'Ryan', 'Gary'], - salary: [623.3, 515.2, 611, 729.0, 843.25], + salary: [623.3, 515.2, 611.0, 729.0, 843.25], score: toBigDecimals(['1.2345', 0.6321, 0.7190, 0.8452, 0.9198]), start_date: toLocalDates('2012-01-01', '2013-09-23', '2014-11-15', '2014-05-11', '2015-03-27')) .types([int, String, BigDecimal, BigDecimal, LocalDate]).build() @@ -444,7 +444,7 @@ class MatrixParquetTest { def dirEx = assertThrows(IllegalArgumentException) { MatrixParquetReader.read(dir) } - assertTrue(dirEx.message.contains('directory')) + assertTrue(dirEx.message.contains('nonempty')) } @Test @@ -739,8 +739,8 @@ class MatrixParquetTest { value: toBigDecimals([12.34, 56.78]) ).types([BigDecimal]).build() - byte[] parquetBytes = MatrixParquetWriter.writeBytes(data, false) - assert parquetBytes != null + assertThrows(IllegalArgumentException) { MatrixParquetWriter.writeBytes(data, false) } + byte[] parquetBytes = MatrixParquetWriter.writeBytes(data, [value: [4, 2] as int[]]) assert parquetBytes.length > 0 } @@ -824,7 +824,7 @@ class MatrixParquetTest { void testWriterBuilderWithPrecisionAndScale() { def data = Matrix.builder('empData').columns( emp_id: 1..5, - salary: [623.3, 515.2, 611, 729.0, 843.25], + salary: [623.3, 515.2, 611.0, 729.0, 843.25], score: toBigDecimals(['1.2345', 0.6321, 0.7190, 0.8452, 0.9198]) ).types([int, BigDecimal, BigDecimal]).build() @@ -899,10 +899,11 @@ class MatrixParquetTest { value: toBigDecimals([12.34, 56.78]) ).types([BigDecimal]).build() - byte[] bytes = MatrixParquetWriter.builder(data) - .inferPrecisionAndScale(false) - .writeBytes() - assert bytes != null + assertThrows(IllegalArgumentException) { + MatrixParquetWriter.builder(data).inferPrecisionAndScale(false).writeBytes() + } + byte[] bytes = MatrixParquetWriter.builder(data).inferPrecisionAndScale(false) + .precision(8).scale(2).writeBytes() assert bytes.length > 0 } diff --git a/matrix-parquet/src/test/groovy/OptimizedParquetTest.groovy b/matrix-parquet/src/test/groovy/OptimizedParquetTest.groovy new file mode 100644 index 000000000..90518d886 --- /dev/null +++ b/matrix-parquet/src/test/groovy/OptimizedParquetTest.groovy @@ -0,0 +1,129 @@ +import static org.junit.jupiter.api.Assertions.* + +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import se.alipsa.matrix.core.Matrix +import se.alipsa.matrix.parquet.* + +import java.time.* + +class OptimizedParquetTest { + @TempDir File directory + + @Test + void externalCdcAndIndexes() { + [0, 1].each { cdc -> + [0, 1].each { index -> + File file = new File('src/test/resources/optimized', "cdc-${cdc}-index-${index}.parquet") + def result = MatrixParquetReader.builder().zoneId('Europe/Stockholm').read(file) + assertEquals(4096, result.rowCount()) + assertEquals(LocalDateTime.of(2024, 3, 31, 1, 30), result[0, 'local']) + assertArrayEquals([0, 0, -1] as byte[], (byte[]) result[0, 'payload']) + assertEquals(1.23G, result[123, 'price']) + assertEquals(['α', null, 'β'], result[1, 'tags']) + assertArrayEquals([1, 0, -1] as byte[], (byte[]) result[1, 'media'].bytes) + assertEquals('image-1.jpg', result[1, 'media'].path) + assertNull(result[2048, 'nulls']) + assertEquals('café-3-' + 'abcd' * 32, result[4095, 'text']) + [true, false].each { useIndex -> + def selected = MatrixParquetReader.builder().columns(['text', 'id']) + .filter(Predicates.ge('id', 3000L)).limit(13).useColumnIndexFilter(useIndex).read(file) + assertEquals(['text', 'id'], selected.columnNames()) + assertEquals((3000L..<3013L).toList(), selected.column('id')) + } + def groups = MatrixParquetReader.builder().rowGroups([1, 0]).limit(3).read(file) + assertEquals([2048L, 2049L, 2050L], groups.column('id')) + MatrixParquetReader.builder().maxBatchRows(127).batches(file).withCloseable { batches -> + int count = 0 + while (batches.hasNext()) { + Matrix batch = batches.next() + assertTrue(batch.rowCount() <= 127) + count += batch.rowCount() + } + assertEquals(4096, count) + } + } + } + } + + @Test + void alternatePairwiseFixtures() { + new File('src/test/resources/optimized').listFiles().findAll { it.name.endsWith('-alternate.parquet') }.each { file -> + def data = MatrixParquetReader.read(file) + assertEquals(4096, data.rowCount()) + assertEquals(1.23G, data[123, 'price']) + assertEquals(['α', null, 'β'], data[1, 'tags']) + } + } + + @Test + @SuppressWarnings('ClassForName') // Characterize JVM array-name lookup; production uses a fixed allowlist. + void fixtureEntryPointsAndLegacyCharacterization() { + File file = new File('src/test/resources/optimized/cdc-1-index-1.parquet') + [MatrixParquetReader.read(file.bytes), MatrixParquetReader.read(new ByteArrayInputStream(file.bytes)), + MatrixParquetReader.read(file.toURI().toURL()), new ParquetFormatProvider().read(file, [:])].each { result -> + assertEquals(4096, result.rowCount()) + assertEquals(['α', null, 'β'], result[1, 'tags']) + assertArrayEquals([0, 0, -1] as byte[], (byte[]) result[0, 'payload']) + } + def legacy = MatrixParquetReader.read(new File('src/test/resources/optimized/matrix-0.7-date-units.parquet')) + assertEquals(1000L, legacy[0, 'value'].time) + assertEquals(2_000_000L, legacy[1, 'value'].time) // Original Timestamp(2000) was written as micros into MILLIS. + def metadata = MatrixParquetReader.builder().zoneId('UTC').read(new File('src/test/resources/optimized/matrix-0.7-metadata.parquet')) + assertEquals(BigDecimal, metadata.type('amount')) + assertEquals(LocalDateTime.of(2024, 1, 1, 0, 0), metadata[0, 'local']) + assertThrows(IllegalArgumentException) { + MatrixParquetReader.read(new File('src/test/resources/optimized/legacy-binary-metadata.parquet')) + } + assertThrows(ClassNotFoundException) { getClass().classLoader.loadClass('[B') } + assertEquals(byte[], Class.forName('[B', false, getClass().classLoader)) + } + + @Test + void canonicalTypesAndBinaryRoundTrip() { + def uuid = UUID.fromString('00112233-4455-6677-8899-aabbccddeeff') + def instant = Instant.parse('1500-01-01T00:00:00.123456Z') + def data = Matrix.builder('policies').columns( + id: [1, 2], text: ['α', null], price: [1.5G, 2.25G], + binary: [[0, -1] as byte[], [] as byte[]], tiny: [(byte) 1, (byte) -2], + small: [(short) 1000, (short) -1000], time: [LocalTime.NOON, null], + instant: [instant, null], uuid: [uuid, null], nested: [[label: 'hello'], [:]]) + .build() + byte[] bytes = MatrixParquetWriter.writeBytes(data) + def result = MatrixParquetReader.read(bytes) + assertEquals(BigDecimal, result.type('price')) + assertEquals([1.50G, 2.25G], result.column('price')) + assertArrayEquals([0, -1] as byte[], (byte[]) result[0, 'binary']) + assertEquals(Byte, result.type('tiny')) + assertEquals((byte) -2, result[1, 'tiny']) + assertEquals(LocalTime.NOON, result[0, 'time']) + assertEquals(instant, result[0, 'instant']) + assertEquals(uuid, result[0, 'uuid']) + assertEquals([label: 'hello'], result[0, 'nested']) + } + + @Test + void validationBeforeWriting() { + File target = new File(directory, 'untouched.parquet') + target.text = 'keep' + [Matrix.builder().columns(value: [1, 'x']).build(), + Matrix.builder().columns(value: [new Date(0), new java.sql.Timestamp(0)]).types(Date).build(), + Matrix.builder().columns(value: [1]).types(BigDecimal).build()].each { data -> + assertThrows(IllegalArgumentException) { MatrixParquetWriter.write(data, target) } + assertEquals('keep', target.text) + } + } + + @Test + void legacyStringsAndWriterTuning() { + def old = MatrixParquetReader.read(new File('src/test/resources/optimized/matrix-0.7-strings.parquet')) + assertEquals('héllo', old[0, 'text']) + assertEquals([label: 'α'], old[0, 'nested']) + def data = Matrix.builder().columns(id: (0..<10000).toList(), text: ['label'] * 10000).build() + File file = new File(directory, 'tuned.parquet') + MatrixParquetWriter.builder(data).pageSize(4096).rowGroupSize(32768).writerVersion('V2') + .bloomFilterEnabled([id: true]).bloomFilterNDV([id: 10000L]).write(file) + assertEquals(10000, MatrixParquetReader.read(file).rowCount()) + } +} diff --git a/matrix-parquet/src/test/groovy/ParquetBaselineTest.groovy b/matrix-parquet/src/test/groovy/ParquetBaselineTest.groovy new file mode 100644 index 000000000..67fd0e25a --- /dev/null +++ b/matrix-parquet/src/test/groovy/ParquetBaselineTest.groovy @@ -0,0 +1,27 @@ +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfSystemProperty + +import se.alipsa.matrix.core.Matrix +import se.alipsa.matrix.parquet.MatrixParquetReader +import se.alipsa.matrix.parquet.MatrixParquetWriter + +/** Opt-in baseline and legacy fixture capture, run before changing the writer. */ +@EnabledIfSystemProperty(named = 'parquet.baseline', matches = 'true') +class ParquetBaselineTest { + @Test + void capture() { + int rows = 1_000_000 + def columns = [id: (0.. columns["text$c".toString()] = (0.. + long start = System.nanoTime() + def result = MatrixParquetReader.read(file) + long millis = (System.nanoTime() - start).intdiv(1_000_000) + assert result.rowCount() == rows + println "baseline wide run=$run rows=$rows fileBytes=${file.length()} millis=$millis" + } + } +} diff --git a/matrix-parquet/src/test/groovy/ParquetCodecProbeTest.groovy b/matrix-parquet/src/test/groovy/ParquetCodecProbeTest.groovy new file mode 100644 index 000000000..3a4d7ef7b --- /dev/null +++ b/matrix-parquet/src/test/groovy/ParquetCodecProbeTest.groovy @@ -0,0 +1,25 @@ +import static org.junit.jupiter.api.Assertions.assertEquals + +import org.junit.jupiter.api.Test + +import se.alipsa.matrix.core.Matrix +import se.alipsa.matrix.parquet.MatrixParquetReader +import se.alipsa.matrix.parquet.MatrixParquetWriter + +/** Normal codec round trips; the opt-in no-Hadoop lane also reports required runtime classes. */ +class ParquetCodecProbeTest { + @Test + void codecs() { + ['SNAPPY', 'ZSTD', 'GZIP', 'UNCOMPRESSED'].each { codec -> + try { + def matrix = Matrix.builder('codec').columns(id: [1, 2], text: ['a', 'b']).build() + byte[] bytes = MatrixParquetWriter.builder(matrix).compressionCodec(codec).writeBytes() + assertEquals([1, 2], MatrixParquetReader.read(bytes).column('id')) + println "CODEC $codec OK" + } catch (NoClassDefFoundError failure) { + if (System.getProperty('parquet.codecSpike') != 'true') { throw failure } + println "CODEC $codec requires ${failure.message}" + } + } + } +} diff --git a/matrix-parquet/src/test/groovy/ParquetDatasetAndHttpTest.groovy b/matrix-parquet/src/test/groovy/ParquetDatasetAndHttpTest.groovy new file mode 100644 index 000000000..92cd1c2af --- /dev/null +++ b/matrix-parquet/src/test/groovy/ParquetDatasetAndHttpTest.groovy @@ -0,0 +1,105 @@ +import static org.junit.jupiter.api.Assertions.* + +import com.sun.net.httpserver.HttpServer +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import se.alipsa.matrix.core.Matrix +import se.alipsa.matrix.parquet.* + +import java.util.concurrent.atomic.AtomicLong + +class ParquetDatasetAndHttpTest { + @TempDir File directory + + @Test + void orderedShardsParallelLimitsAndSchemaChecks() { + File first = new File(directory, 'a.parquet') + File second = new File(directory, 'b.parquet') + [first, second].eachWithIndex { file, index -> + MatrixParquetWriter.write(Matrix.builder().columns(id: [index * 2, index * 2 + 1]).build(), file) + } + [1, 2].each { workers -> + def data = MatrixParquetReader.builder().parallelism(workers).sourceFileColumn('source') + .limit(3).read([second, first]) + assertEquals([2, 3, 0], data.column('id')) + assertEquals([second.canonicalPath, second.canonicalPath, first.canonicalPath], data.column('source')) + } + def early = MatrixParquetReader.builder().parallelism(2).maxBatchRows(1).batches([first, second]) + assertEquals([0], early.next().column('id')) + early.close() + assertFalse(early.hasNext()) + early.close() + assertEquals([0, 1, 2, 3], MatrixParquetReader.read(directory).column('id')) + assertEquals([0, 1, 2, 3], MatrixParquetReader.readGlob("${directory.absolutePath}/*.parquet").column('id')) + assertEquals(['id'], MatrixParquetReader.builder().limit(0).read([first, second]).columnNames()) + assertEquals(0, MatrixParquetReader.builder().filter(Predicates.lt('id', 0)).read([first, second]).rowCount()) + def perFile = MatrixParquetReader.builder().parallelism(2) + .rowGroupsByFile([(first.canonicalPath): [], (second.canonicalPath): [0]]).read([first, second]) + assertEquals([2, 3], perFile.column('id')) + assertThrows(IllegalArgumentException) { MatrixParquetReader.read([first, first]) } + MatrixParquetWriter.write(Matrix.builder().columns(id: ['incompatible']).build(), second) + assertThrows(IllegalArgumentException) { MatrixParquetReader.read([first, second]) } + } + + @Test + void parallelRowGroupsKeepOrderAndMatchingLimit() { + File file = new File('src/test/resources/optimized/cdc-1-index-1.parquet') + [1, 2].each { workers -> + def result = MatrixParquetReader.builder().parallelism(workers).columns(['id']) + .filter(Predicates.ge('id', 2040L)).maxBatchRows(7).limit(20).read([file]) + assertEquals((2040L..<2060L).toList(), result.column('id')) + } + } + + @Test + void rangeReadsFallbackAndObjectChange() { + File file = new File(directory, 'large.parquet') + def data = Matrix.builder().columns(id: (0..<20000).toList(), text: (0..<20000).collect { UUID.randomUUID().toString() }).build() + MatrixParquetWriter.builder(data).compressionCodec('UNCOMPRESSED').pageSize(4096).dictionaryEncoding(false).write(file) + byte[] bytes = file.bytes + AtomicLong transferred = new AtomicLong() + boolean changed = false + def server = HttpServer.create(new InetSocketAddress('127.0.0.1', 0), 0) + server.createContext('/file') { exchange -> + String range = exchange.requestHeaders.getFirst('Range') + exchange.responseHeaders.add('ETag', changed ? '"changed"' : '"original"') + if (range != null) { + def match = range =~ /bytes=(\d+)-(\d+)/ + assertTrue(match.matches()) + int start = Integer.parseInt(match.group(1)) + int end = Integer.parseInt(match.group(2)) + exchange.responseHeaders.add('Content-Range', "bytes $start-$end/${bytes.length}") + exchange.sendResponseHeaders(206, end - start + 1) + exchange.responseBody.write(bytes, start, end - start + 1) + transferred.addAndGet(end - start + 1) + } else { + exchange.sendResponseHeaders(200, bytes.length) + exchange.responseBody.write(bytes) + transferred.addAndGet(bytes.length) + } + exchange.close() + } + server.createContext('/download') { exchange -> + exchange.sendResponseHeaders(200, bytes.length) + exchange.responseBody.write(bytes) + exchange.close() + } + server.start() + try { + URL url = new URI("http://127.0.0.1:${server.address.port}/file").toURL() + def selected = MatrixParquetReader.builder().remoteReadMode('RANGE').columns(['id']) + .filter(Predicates.ge('id', 19990)).read(url) + assertEquals((19990..<20000).toList(), selected.column('id')) + println "HTTP bytes full=${bytes.length} transferred=${transferred.get()}" + assertTrue(transferred.get() < bytes.length, "Transferred ${transferred.get()} of ${bytes.length}") + URL download = new URI("http://127.0.0.1:${server.address.port}/download").toURL() + assertEquals(2, MatrixParquetReader.builder().remoteReadMode('AUTO').limit(2).read(download).rowCount()) + assertThrows(IOException) { MatrixParquetReader.builder().remoteReadMode('RANGE').read(download) } + def options = new ParquetReadOptions(remoteReadMode: 'RANGE') + def input = new HttpRangeInputFile(url, options) + changed = true + assertThrows(IOException) { input.newStream().withCloseable { it.read() } } + } finally { server.stop(0) } + } +} diff --git a/matrix-parquet/src/test/groovy/ParquetInteropTest.groovy b/matrix-parquet/src/test/groovy/ParquetInteropTest.groovy new file mode 100644 index 000000000..42d764903 --- /dev/null +++ b/matrix-parquet/src/test/groovy/ParquetInteropTest.groovy @@ -0,0 +1,40 @@ +import static org.junit.jupiter.api.Assertions.* +import static org.junit.jupiter.api.Assumptions.assumeTrue + +import org.junit.jupiter.api.Test + +import se.alipsa.matrix.core.Matrix +import se.alipsa.matrix.parquet.MatrixParquetReader +import se.alipsa.matrix.parquet.MatrixParquetWriter + +class ParquetInteropTest { + @Test + void exportAndReadRewrittenFiles() { + File directory = new File('build/test-interop') + directory.mkdirs() + def data = Matrix.builder('interop').columns(text: ['héllo', null], + payload: [[0, -1] as byte[], [] as byte[]], price: [1.25G, -2.50G], + tags: [['α', null], []]).build() + ['SNAPPY', 'ZSTD'].each { codec -> + File file = new File(directory, "${codec}.parquet") + MatrixParquetWriter.builder(data).compressionCodec(codec).write(file) + File rewritten = new File(directory, "${codec}.cdc.parquet") + if (rewritten.isFile()) { + def result = MatrixParquetReader.read(rewritten) + assertEquals(['héllo', null], result.column('text')) + assertArrayEquals([0, -1] as byte[], (byte[]) result[0, 'payload']) + assertEquals([1.25G, -2.50G], result.column('price')) + assertEquals([['α', null], []], result.column('tags')) + } + } + } + + @Test + void defaultCdcFixtureWhenGenerated() { + File file = new File('build/default-cdc/default-cdc.parquet') + assumeTrue(file.isFile(), 'SKIP default CDC: enable -PrunSlowTests=true with pinned PyArrow 21.0.0') + def data = MatrixParquetReader.read(file) + assertEquals(16384, data.rowCount()) + assertEquals('café-1-' + 'abcd' * 32, data[1, 'text']) + } +} diff --git a/matrix-parquet/src/test/groovy/ParquetLegacyCaptureTest.groovy b/matrix-parquet/src/test/groovy/ParquetLegacyCaptureTest.groovy new file mode 100644 index 000000000..2a660fc5b --- /dev/null +++ b/matrix-parquet/src/test/groovy/ParquetLegacyCaptureTest.groovy @@ -0,0 +1,57 @@ +import static org.junit.jupiter.api.Assertions.* + +import groovy.transform.CompileStatic + +import org.codehaus.groovy.control.CompilerConfiguration +import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfSystemProperty + +import se.alipsa.matrix.core.Matrix + +import java.sql.Timestamp +import java.time.LocalDateTime +import java.time.ZoneId +import java.time.ZoneOffset + +/** Reproduce historical fixtures with the original writer isolated from current classes. */ +@EnabledIfSystemProperty(named = 'parquet.captureLegacy', matches = 'true') +class ParquetLegacyCaptureTest { + private static final String REVISION = 'b2095159664b34ce426685c387adf0bccdb33e01' + + @Test + void captureOriginalWriter() { + def configuration = new CompilerConfiguration() + configuration.addCompilationCustomizers(new ASTTransformationCustomizer(CompileStatic)) + new GroovyClassLoader(getClass().classLoader, configuration).withCloseable { loader -> + List sources = ['ParquetValueWriter', 'MatrixParquetWriter'].collect { name -> + String path = "matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/${name}.groovy" + def process = new ProcessBuilder('git', 'show', "$REVISION:$path").start() + String source = process.inputStream.getText('UTF-8') + assertEquals(0, process.waitFor()) + source + } + Set imports = [] + List bodies = sources.collect { source -> + source.readLines().findAll { line -> + if (line.startsWith('import ')) { imports.add(line); return false } + !line.startsWith('package ') + }.join('\n') + } + loader.parseClass('package se.alipsa.matrix.parquet\n' + imports.join('\n') + '\n' + bodies.join('\n'), 'LegacyWriter.groovy') + Class writer = loader.loadClass('se.alipsa.matrix.parquet.MatrixParquetWriter') + def write = writer.getMethod('write', Matrix, File, boolean) + File root = new File('src/test/resources/optimized') + def dates = Matrix.builder('legacyDates').columns(value: [new Date(1000L), new Timestamp(2000L)]).types(Date).build() + write.invoke(null, dates, new File(root, 'matrix-0.7-date-units.parquet'), true) + def metadata = Matrix.builder('legacyMetadata').columns(amount: [1.25G, -2.50G], + local: [LocalDateTime.of(2024, 1, 1, 0, 0), null]).types(BigDecimal, LocalDateTime).build() + writer.getMethod('write', Matrix, File, ZoneId).invoke(null, metadata, new File(root, 'matrix-0.7-metadata.parquet'), ZoneOffset.UTC) + def failure = assertThrows(java.lang.reflect.InvocationTargetException) { + write.invoke(null, Matrix.builder().columns(value: [[0, -1] as byte[]]).types(byte[]).build(), + new File('build/legacy-invalid-bytes.parquet'), true) + } + assertEquals('org.apache.parquet.schema.InvalidSchemaException', failure.cause.getClass().name) + } + } +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/CountingInput.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/CountingInput.groovy new file mode 100644 index 000000000..d4b82ab0f --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/CountingInput.groovy @@ -0,0 +1,32 @@ +package se.alipsa.matrix.parquet + +import org.apache.parquet.io.InputFile +import org.apache.parquet.io.SeekableInputStream + +import java.nio.ByteBuffer + +/** Test-only exact byte counter over an independent seekable input. */ +class CountingInput implements InputFile { + final InputFile delegate + long totalBytes + CountingInput(InputFile delegate) { this.delegate = delegate } + @Override long getLength() { delegate.length } + @Override SeekableInputStream newStream() { + SeekableInputStream stream = delegate.newStream() + new SeekableInputStream() { + @Override long getPos() { stream.pos } + @Override void seek(long value) { stream.seek(value) } + @Override int read() { int value = stream.read(); if (value >= 0) { totalBytes++ } ; value } + @Override int read(byte[] target, int offset, int length) { + int count = stream.read(target, offset, length) + if (count > 0) { totalBytes += count } + count + } + @Override void readFully(byte[] target) { readFully(target, 0, target.length) } + @Override void readFully(byte[] target, int offset, int length) { stream.readFully(target, offset, length); totalBytes += length } + @Override int read(ByteBuffer target) { int count = stream.read(target); if (count > 0) { totalBytes += count } ; count } + @Override void readFully(ByteBuffer target) { int count = target.remaining(); stream.readFully(target); totalBytes += count } + @Override void close() { stream.close() } + } + } +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/HttpTransportTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/HttpTransportTest.groovy new file mode 100644 index 000000000..f310571ac --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/HttpTransportTest.groovy @@ -0,0 +1,97 @@ +package se.alipsa.matrix.parquet + +import static org.junit.jupiter.api.Assertions.* + +import org.junit.jupiter.api.Test + +class HttpTransportTest { + @Test + void hubTokenIsOriginScopedAcrossRedirects() { + List requests = [] + URLStreamHandler handler = new URLStreamHandler() { + @Override + protected URLConnection openConnection(URL url) { + def connection = new StubConnection(url, requests.empty ? 302 : 200, + requests.empty ? [Location: 'https://cdn.example/signed'] : [:]) + requests.add(connection) + connection + } + } + URL source = new URL(null, 'https://huggingface.co/datasets/owner/repo/resolve/main/file', handler) + def options = new ParquetReadOptions() + options.hubToken = 'test-secret' + HttpTransport.open(source, options, [:]).disconnect() + assertEquals('Bearer test-secret', requests[0].getRequestProperty('Authorization')) + assertNull(requests[1].getRequestProperty('Authorization')) + assertFalse(options.toMap().containsKey('hubToken')) + } + + @Test + void downgradeAndMalformedRangeFailAndDisconnect() { + StubConnection request + URL source = new URL(null, 'https://huggingface.co/file', new URLStreamHandler() { + @Override protected URLConnection openConnection(URL url) { + request = new StubConnection(url, 302, [Location: 'http://huggingface.co/file']) + request + } + }) + assertThrows(IOException) { HttpTransport.open(source, new ParquetReadOptions(), [:]) } + assertTrue(request.disconnected) + URL malformed = new URL(null, 'https://example.test/file', new URLStreamHandler() { + @Override protected URLConnection openConnection(URL url) { + request = new StubConnection(url, 206, ['Content-Range': 'bytes 0-1/10']) + request + } + }) + assertThrows(IOException) { new HttpRangeInputFile(malformed, new ParquetReadOptions()) } + assertTrue(request.disconnected) + } + + @Test + void truncatedRangeBodyFailsAndDisconnects() { + List requests = [] + URL source = new URL(null, 'https://example.test/file', new URLStreamHandler() { + @Override protected URLConnection openConnection(URL url) { + boolean probe = requests.empty + def connection = new StubConnection(url, 206, ['Content-Range': probe ? 'bytes 0-0/12' : 'bytes 0-3/12']) + requests.add(connection) + connection + } + }) + def input = new HttpRangeInputFile(source, new ParquetReadOptions(rangeCacheBytes: 4)) + assertThrows(EOFException) { input.newStream().withCloseable { it.readFully(new byte[4]) } } + assertTrue(requests.every { it.disconnected }) + } + + @Test + void connectionRetriesAreBounded() { + int attempts = 0 + URL source = new URL(null, 'https://example.test/file', new URLStreamHandler() { + @Override protected URLConnection openConnection(URL url) { + attempts++ + new StubConnection(url, 0, [:]) { + @Override int getResponseCode() { throw new IOException('Transient connect failure') } + } + } + }) + assertThrows(IOException) { HttpTransport.open(source, new ParquetReadOptions(), [:]) } + assertEquals(2, attempts) + } + + private static class StubConnection extends HttpURLConnection { + private final int status + private final Map headers + boolean disconnected + StubConnection(URL url, int status, Map headers) { + super(url) + this.status = status + this.headers = headers + } + @Override int getResponseCode() { status } + @Override String getHeaderField(String key) { headers[key] } + @Override void disconnect() { disconnected = true } + @Override boolean usingProxy() { false } + @Override void connect() {} + @Override InputStream getInputStream() { new ByteArrayInputStream([0] as byte[]) } + } +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/LiveHeapSampler.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/LiveHeapSampler.groovy new file mode 100644 index 000000000..506ffa7cb --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/LiveHeapSampler.groovy @@ -0,0 +1,43 @@ +package se.alipsa.matrix.parquet + +import java.lang.management.ManagementFactory + +/** Test-only maximum live-heap sampler using post-GC notifications. */ +class LiveHeapSampler { + final java.util.concurrent.atomic.AtomicLong peak = new java.util.concurrent.atomic.AtomicLong() + private final List emitters = [] + private final Set heapPools = ManagementFactory.memoryPoolMXBeans.findAll { it.type == java.lang.management.MemoryType.HEAP }*.name.toSet() + private final Map initialCounts = [:] + private final java.util.concurrent.CountDownLatch finished = new java.util.concurrent.CountDownLatch(1) + private volatile Map finalCounts + private final javax.management.NotificationListener listener = { notification, handback -> + if (notification.type == com.sun.management.GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION) { + def event = com.sun.management.GarbageCollectionNotificationInfo.from((javax.management.openmbean.CompositeData) notification.userData) + def info = event.gcInfo + if (info.id > (initialCounts[event.gcName] ?: 0L)) { + long used = info.memoryUsageAfterGc.findAll { String pool, usage -> heapPools.contains(pool) }.values().sum { it.used } as long + peak.accumulateAndGet(used, { long a, long b -> Math.max(a, b) } as java.util.function.LongBinaryOperator) + Map target = finalCounts + if (target != null && info.id > (target[event.gcName] ?: 0L)) { finished.countDown() } + } + } + } as javax.management.NotificationListener + + LiveHeapSampler() { + ManagementFactory.garbageCollectorMXBeans.each { collector -> + if (collector instanceof javax.management.NotificationEmitter) { + initialCounts[collector.name] = collector.collectionCount + collector.addNotificationListener(listener, null, null) + emitters.add(collector) + } + } + } + + @SuppressWarnings('ExplicitGarbageCollection') + void finish() { + finalCounts = ManagementFactory.garbageCollectorMXBeans.collectEntries { [(it.name): it.collectionCount] } + System.gc() + assert finished.await(5, java.util.concurrent.TimeUnit.SECONDS) : 'Final GC notification was not observed' + emitters.each { it.removeNotificationListener(listener) } + } +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetDatasetBenchmarkTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetDatasetBenchmarkTest.groovy new file mode 100644 index 000000000..12aa443cf --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetDatasetBenchmarkTest.groovy @@ -0,0 +1,38 @@ +package se.alipsa.matrix.parquet + +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfSystemProperty + +import se.alipsa.matrix.core.Matrix + +/** Opt-in bounded shard-decoding measurements, without timing assertions. */ +@EnabledIfSystemProperty(named = 'parquet.datasetBenchmark', matches = 'true') +class ParquetDatasetBenchmarkTest { + @Test + @SuppressWarnings('ExplicitGarbageCollection') + void compareWorkers() { + File directory = new File('build/dataset-benchmark') + directory.mkdirs() + List files = (0..<4).collect { int shard -> + File file = new File(directory, "shard-${shard}.parquet") + int start = shard * 250000 + MatrixParquetWriter.write(Matrix.builder().columns(id: (start.. + (0..<4).each { run -> + System.gc() + def sampler = new LiveHeapSampler() + long start = System.nanoTime() + def result = MatrixParquetReader.builder().parallelism(workers).maxBatchRows(65536).read(files) + long millis = (System.nanoTime() - start).intdiv(1_000_000) + sampler.finish() + assert result.rowCount() == 1_000_000 + assert result[0, 'id'] == 0 + assert result[999999, 'id'] == 999999 + println "DATASET workers=$workers run=$run rows=${result.rowCount()} millis=$millis peakLiveAfterGc=${sampler.peak.get()}" + } + } + } +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetLayoutBenchmarkTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetLayoutBenchmarkTest.groovy new file mode 100644 index 000000000..7d11a1456 --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetLayoutBenchmarkTest.groovy @@ -0,0 +1,35 @@ +package se.alipsa.matrix.parquet + +import org.apache.parquet.io.LocalInputFile +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfSystemProperty + +/** Opt-in full/projection/filter measurements against externally generated identical layouts. */ +@EnabledIfSystemProperty(named = 'parquet.layoutBenchmark', matches = 'true') +class ParquetLayoutBenchmarkTest { + @Test + @SuppressWarnings('ExplicitGarbageCollection') + void compareLayouts() { + ['ordinary', 'indexed', 'cdc-indexed'].each { layout -> + File file = new File("build/layout-benchmarks/${layout}.parquet") + assert file.isFile() : 'Run src/test/python/generate_benchmarks.py first' + ['full', 'projection', 'selective'].each { mode -> + (0..<4).each { int run -> + System.gc() + def options = new ParquetReadOptions() + if (mode != 'full') { options.columns = ['id'] } + if (mode == 'selective') { options.filter = Predicates.ge('id', 999900L) } + def input = new CountingInput(new LocalInputFile(file.toPath())) + def sampler = new LiveHeapSampler() + long start = System.nanoTime() + def result = MatrixParquetReader.readInput(input, options, null) + long elapsed = System.nanoTime() - start + sampler.finish() + assert result.rowCount() == (mode == 'selective' ? 100 : 1000000) + assert result[result.rowCount() - 1, 'id'] == 999999L + println "LAYOUT layout=$layout mode=$mode run=$run rows=${result.rowCount()} fileBytes=${file.length()} readBytes=${input.totalBytes} millis=${elapsed.intdiv(1000000)} peakLiveAfterGc=${sampler.peak.get()}" + } + } + } + } +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetPruningTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetPruningTest.groovy new file mode 100644 index 000000000..3445c0b74 --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetPruningTest.groovy @@ -0,0 +1,128 @@ +package se.alipsa.matrix.parquet + +import static org.junit.jupiter.api.Assertions.* + +import org.apache.parquet.conf.PlainParquetConfiguration +import org.apache.parquet.hadoop.ParquetFileReader +import org.apache.parquet.io.LocalInputFile +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import se.alipsa.matrix.core.Matrix + + +class ParquetPruningTest { + @TempDir File directory + + @Test + void fixturesHavePhysicalIndexesAndMultiplePages() { + [0, 1].each { cdc -> + [0, 1].each { indexed -> + File file = new File('src/test/resources/optimized', "cdc-${cdc}-index-${indexed}.parquet") + def options = org.apache.parquet.ParquetReadOptions.builder(new PlainParquetConfiguration()).build() + ParquetFileReader.open(new LocalInputFile(file.toPath()), options).withCloseable { reader -> + assertEquals(2, reader.rowGroups.size()) + def column = reader.rowGroups[0].columns.find { it.path.toDotString() == 'id' } + assertEquals(indexed == 1, column.columnIndexReference != null) + assertEquals(indexed == 1, column.offsetIndexReference != null) + if (indexed == 1) { checkIndexes(reader, column, file.length(), reader.rowGroups[0].rowCount) } + } + } + } + } + + private static void checkIndexes(ParquetFileReader reader, column, long length, long rows) { + def offsets = reader.readOffsetIndex(column) + def bounds = reader.readColumnIndex(column) + assertTrue(offsets.pageCount > 1) + assertEquals(offsets.pageCount, bounds.nullPages.size()) + assertEquals(0L, offsets.getFirstRowIndex(0)) + (0.. + assertTrue(offsets.getOffset(page) >= 4) + assertTrue(offsets.getOffset(page) + offsets.getCompressedPageSize(page) <= length) + assertTrue(offsets.getFirstRowIndex(page) < rows) + if (page > 0) { + assertTrue(offsets.getOffset(page) > offsets.getOffset(page - 1)) + assertTrue(offsets.getFirstRowIndex(page) > offsets.getFirstRowIndex(page - 1)) + } + } + } + + @Test + void writerIndexesAcrossOutputPaths() { + def data = Matrix.builder().columns(id: (0..<20000).toList(), text: ['label'] * 20000).build() + File file = new File(directory, 'indexes.parquet') + def writer = MatrixParquetWriter.builder(data).pageSize(1024).dictionaryEncoding(false) + writer.write(file) + def stream = new ByteArrayOutputStream() + writer.write(stream) + [new LocalInputFile(file.toPath()), new InMemoryInputFile(writer.writeBytes()), + new InMemoryInputFile(stream.toByteArray())].each { input -> + def options = org.apache.parquet.ParquetReadOptions.builder(new PlainParquetConfiguration()).build() + ParquetFileReader.open(input, options).withCloseable { reader -> + reader.rowGroups[0].columns.each { column -> checkIndexes(reader, column, input.length, reader.rowGroups[0].rowCount) } + } + } + } + + @Test + void pageIndexesReduceDataReadsWithoutRowGroupPruning() { + File file = new File(directory, 'pages.parquet') + def data = Matrix.builder().columns(id: (0..<20000).toList(), text: ['categorical'] * 20000).build() + MatrixParquetWriter.builder(data).compressionCodec('UNCOMPRESSED').dictionaryEncoding(false) + .pageSize(1024).rowGroupSize(16 * 1024 * 1024L).write(file) + List counts = [] + [false, true].each { enabled -> + def input = new CountingInput(new LocalInputFile(file.toPath())) + def options = new ParquetReadOptions(columns: ['id'], filter: Predicates.ge('id', 19990), useColumnIndexFilter: enabled) + options.useRowGroupFilters = false + def result = MatrixParquetReader.readInput(input, options, null) + assertEquals((19990..<20000).toList(), result.column('id')) + counts.add(input.totalBytes) + } + println "PAGE_INDEX bytes full=${counts[0]} pruned=${counts[1]}" + assertTrue(counts[1] < counts[0], "Indexed totalBytes ${counts[1]} versus full ${counts[0]}") + } + + @Test + void rejectedRowsNeverReachColumnsAndDictionaryStringsAreReused() { + File file = new File(directory, 'dictionary.parquet') + MatrixParquetWriter.write(Matrix.builder().columns(id: [1, 2, 3, 4, 5], text: ['same'] * 5).build(), file) + def result = MatrixParquetReader.read(file) + assertSame(result[0, 'text'], result[4, 'text']) + [Predicates.ge('id', 3), Predicates.ne('id', 3), Predicates.not(Predicates.ge('id', 1))].each { predicate -> + def filtered = MatrixParquetReader.builder().filter(predicate).read(file) + assertEquals(filtered.column('id').size(), filtered.column('text').size()) + assertEquals((1..5).findAll { predicate.test([id: it]) }, filtered.column('id')) + } + def empty = MatrixParquetReader.builder().rowGroups([0]).filter(Predicates.gt('id', 100)).read(file) + assertEquals(['id', 'text'], empty.columnNames()) + assertEquals(0, empty.rowCount()) + } + + @Test + void bloomFilterSkipsAnInteriorAbsentKeyIndependently() { + File file = new File(directory, 'bloom.parquet') + def data = Matrix.builder().columns(id: (0..<10000).collect { 2 * it + 1 }).build() + MatrixParquetWriter.builder(data).compressionCodec('UNCOMPRESSED').dictionaryEncoding(false) + .rowGroupSize(16L * 1024 * 1024).bloomFilterEnabled([id: true]).bloomFilterNDV([id: 10000L]).write(file) + int absent + ParquetFileReader.open(new LocalInputFile(file.toPath()), org.apache.parquet.ParquetReadOptions.builder(new PlainParquetConfiguration()).build()).withCloseable { reader -> + assertEquals(1, reader.rowGroups.size()) + def bloom = reader.readBloomFilter(reader.rowGroups[0].columns[0]) + assertNotNull(bloom) + absent = (2..<19998).find { it % 2 == 0 && !bloom.findHash(bloom.hash(it)) } + } + List counts = [] + [false, true].each { enabled -> + def input = new CountingInput(new LocalInputFile(file.toPath())) + def options = new ParquetReadOptions(filter: Predicates.eq('id', absent), useColumnIndexFilter: false) + options.useRowGroupFilters = enabled + assertEquals(0, MatrixParquetReader.readInput(input, options, null).rowCount()) + counts.add(input.totalBytes) + } + assertTrue(counts[1] < counts[0], "Bloom totalBytes ${counts[1]} versus full ${counts[0]}") + println "BLOOM bytes full=${counts[0]} pruned=${counts[1]}" + } + +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReadBenchmarkTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReadBenchmarkTest.groovy new file mode 100644 index 000000000..4ff865b5b --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReadBenchmarkTest.groovy @@ -0,0 +1,74 @@ +package se.alipsa.matrix.parquet + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path +import org.apache.parquet.example.data.Group +import org.apache.parquet.hadoop.ParquetFileReader +import org.apache.parquet.hadoop.ParquetReader +import org.apache.parquet.hadoop.example.GroupReadSupport +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.EnabledIfSystemProperty + +import se.alipsa.matrix.core.Matrix + +import java.lang.management.ManagementFactory +import java.time.LocalDate + +/** Opt-in comparison against the pre-change Group loop using shared schema conversions. */ +@EnabledIfSystemProperty(named = 'parquet.baseline', matches = 'true') +class ParquetReadBenchmarkTest { + @Test + @SuppressWarnings('ExplicitGarbageCollection') // Opt-in retained-heap benchmark, not production behavior. + void compare() { + int rows = Integer.parseInt(System.getProperty('parquet.benchmarkRows', '1000000')) + ['wide', 'categorical', 'mixed'].each { workload -> + def columns = [id: (0.. + columns["text$c".toString()] = (0.. + (0..<4).each { int run -> + System.gc() + long beforeHeap = ManagementFactory.memoryMXBean.heapMemoryUsage.used + def bean = (com.sun.management.ThreadMXBean) ManagementFactory.threadMXBean + long allocated = bean.getThreadAllocatedBytes(Thread.currentThread().threadId()) + def sampler = new LiveHeapSampler() + long start = System.nanoTime() + Matrix result = columnar ? MatrixParquetReader.read(file) : legacy(file, types) + long elapsed = System.nanoTime() - start + allocated = bean.getThreadAllocatedBytes(Thread.currentThread().threadId()) - allocated + sampler.finish() + long retained = ManagementFactory.memoryMXBean.heapMemoryUsage.used - beforeHeap + assert result.rowCount() == rows + assert result[rows - 1, 'id'] == rows - 1 + println "BENCH workload=$workload columnar=$columnar run=$run rows=$rows bytes=${file.length()} millis=${elapsed.intdiv(1_000_000)} allocated=$allocated retained=$retained peakLiveAfterGc=${sampler.peak.get()}" + } + } + } + } + + private static Matrix legacy(File file, List types) { + Path path = new Path(file.toURI()) + def schema = ParquetFileReader.readFooter(new Configuration(), path).fileMetaData.schema + def names = schema.fields*.name + ParquetReader.builder(new GroupReadSupport(), path).build().withCloseable { reader -> + def builder = Matrix.builder('benchmark').columnNames(names).types(types) + Group row = reader.read() + while (row != null) { + def values = [] + names.eachWithIndex { String name, int index -> values << MatrixParquetReader.readValue(row, name, schema.getType(name), types[index]) } + builder.addRow(values as Object[]) + row = reader.read() + } + builder.build() + } + } +} diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetTypePolicyTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetTypePolicyTest.groovy new file mode 100644 index 000000000..b7f9824f5 --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetTypePolicyTest.groovy @@ -0,0 +1,247 @@ +package se.alipsa.matrix.parquet + +import static org.junit.jupiter.api.Assertions.* + +import org.apache.parquet.conf.PlainParquetConfiguration +import org.apache.parquet.example.data.simple.SimpleGroupFactory +import org.apache.parquet.hadoop.example.ExampleParquetWriter +import org.apache.parquet.io.LocalOutputFile +import org.apache.parquet.io.api.Binary +import org.apache.parquet.schema.LogicalTypeAnnotation as Logical +import org.apache.parquet.schema.MessageTypeParser +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName as Physical +import org.apache.parquet.schema.Types +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import se.alipsa.matrix.core.Matrix + +import java.sql.Timestamp +import java.time.* +import java.time.temporal.Temporal + +class ParquetTypePolicyTest { + @TempDir File directory + + private enum Category { + FIRST + @Override String toString() { 'display label' } + } + + @Test + void everyWriterMetadataNameIsAllowlistedAndCanonical() { + Map examples = [(String): 'hello', (Integer): 1, (Long): 1L, (Float): 1F, + (Double): 1D, (Boolean): true, (Byte): (byte) 1, (Short): (short) 1, + (BigInteger): BigInteger.TEN, (BigDecimal): 1.25G, (Character): 'x' as char, + (byte[]): [0, -1] as byte[], (LocalDate): LocalDate.of(2024, 1, 1), + (LocalDateTime): LocalDateTime.of(2024, 1, 1, 0, 0), (LocalTime): LocalTime.of(1, 2, 3, 456789123), + (Instant): Instant.parse('2024-01-01T00:00:00.123456Z'), (UUID): UUID.randomUUID(), + (Date): new Date(0), (java.sql.Date): java.sql.Date.valueOf('2024-01-01'), + (java.sql.Time): java.sql.Time.valueOf('12:00:00'), (Timestamp): Timestamp.from(Instant.EPOCH), + (List): ['label', null], (Map): [label: 'value'], (Category): Category.FIRST] + examples.each { Class declaration, Object value -> + def matrix = Matrix.builder('types').columns(value: [value, null]).types(declaration).build() + byte[] bytes = MatrixParquetWriter.writeBytes(matrix) + File file = new File(directory, declaration.simpleName + '.parquet') + def spi = new ParquetFormatProvider() + spi.write(matrix, file, [:]) + def stream = new ByteArrayOutputStream() + MatrixParquetWriter.builder(matrix).write(stream) + [MatrixParquetReader.read(bytes), spi.read(file, [:]), + MatrixParquetReader.read(new ByteArrayInputStream(stream.toByteArray()))].each { result -> + assertTrue(ColumnTypePolicy.METADATA_TYPES.containsKey(result.type('value').name)) + if (declaration == Category) { assertEquals('FIRST', result[0, 'value']) } + else if (declaration == Character) { assertEquals('x', result[0, 'value']) } + else if (declaration == byte[]) { assertArrayEquals((byte[]) value, (byte[]) result[0, 'value']) } + else { assertEquals(value, result[0, 'value'], "Round trip $declaration") } + assertNull(result[1, 'value']) + } + } + } + + @Test + void supertypesAllNullAndConcreteMismatch() { + [(Number): 1, (CharSequence): 'a', (Comparable): 1, (Serializable): 1, + (Temporal): LocalDate.of(2024, 1, 1), (Object): 1.25G].each { Class type, Object value -> + def result = MatrixParquetReader.read(MatrixParquetWriter.writeBytes( + Matrix.builder().columns(value: [value, null]).types(type).build())) + assertEquals(value.getClass(), result.type('value')) + } + [Number, Temporal].each { Class type -> + assertThrows(IllegalArgumentException) { MatrixParquetWriter.writeBytes(Matrix.builder().columns(value: [null]).types(type).build()) } + } + def empty = MatrixParquetReader.read(MatrixParquetWriter.writeBytes(Matrix.builder().columns(value: [null]).build())) + assertEquals(String, empty.type('value')) + [ZonedDateTime.now(), OffsetDateTime.now(), new Object()].each { value -> + assertThrows(IllegalArgumentException) { MatrixParquetWriter.writeBytes(Matrix.builder().columns(value: [value]).build()) } + } + [Instant.MIN, Instant.MAX, Instant.EPOCH.plusNanos(1)].each { instant -> + assertThrows(IllegalArgumentException) { MatrixParquetWriter.writeBytes(Matrix.builder().columns(value: [instant]).build()) } + } + } + + @Test + void temporalSubclassPoliciesAreValidatedBeforeOutput() { + File target = new File(directory, 'dates.parquet') + target.text = 'preserve' + [[new Date(1000), new Timestamp(2000)], [new java.sql.Date(1000), new Timestamp(2000)]].each { pair -> + [pair, pair.reverse()].each { values -> + def failure = assertThrows(IllegalArgumentException) { + MatrixParquetWriter.write(Matrix.builder().columns(value: values).types(Date).build(), target) + } + assertTrue(failure.message.contains('row 1')) + assertEquals('preserve', target.text) + } + } + [new Timestamp(2000), java.sql.Date.valueOf('2024-01-01'), java.sql.Time.valueOf('12:00:00')].each { value -> + def result = MatrixParquetReader.read(MatrixParquetWriter.writeBytes( + Matrix.builder().columns(value: [value]).types(Date).build())) + assertEquals(Date, result.type('value')) + assertEquals(value.time, result[0, 'value'].time) + } + } + + @Test + void decimalsUseNestedAndUniformSettingsWithoutDoubleFallback() { + def matrix = Matrix.builder().columns(price: [1.5G, 2.25G], nested: [[[amount: 1.25G]], [[amount: 2.75G]]]).types(Number, List).build() + def options = new ParquetWriteOptions().inferPrecisionAndScale(false).precision(8).scale(2) + .decimalMeta(['nested.element.amount': [6, 3] as int[]]) + def result = MatrixParquetReader.read(MatrixParquetWriter.writeBytes(matrix, options)) + assertEquals(BigDecimal, result.type('price')) + assertEquals(1.50G, result[0, 'price']) + assertEquals([[amount: 1.250G]], result[0, 'nested']) + assertThrows(IllegalArgumentException) { MatrixParquetWriter.writeBytes(matrix, new ParquetWriteOptions().inferPrecisionAndScale(false)) } + assertThrows(IllegalArgumentException) { MatrixParquetWriter.writeBytes(matrix, new ParquetWriteOptions().decimalMeta([missing: [6, 2] as int[]])) } + def precise = 12345678901234567890.123456789012345678G + def exact = MatrixParquetReader.read(MatrixParquetWriter.writeBytes(Matrix.builder().columns(value: [precise]).build())) + assertEquals(precise, exact[0, 'value']) + def allNull = MatrixParquetReader.read(MatrixParquetWriter.writeBytes(Matrix.builder().columns(value: [null]).types(BigDecimal).build())) + assertEquals(BigDecimal, allNull.type('value')) + assertNull(allNull[0, 'value']) + def negative = Matrix.builder().columns(value: [new BigDecimal('1E+9')]).build() + assertEquals(new BigDecimal('1000000000'), MatrixParquetReader.read(MatrixParquetWriter.writeBytes(negative))[0, 'value']) + } + + @Test + void externalIntegerDecimalsAndTimestampUnits() { + def builder = Types.buildMessage() + .optional(Physical.INT32).as(Logical.decimalType(2, 9)).named('small') + .optional(Physical.INT64).as(Logical.decimalType(4, 18)).named('large') + Logical.TimeUnit.values().each { unit -> + [false, true].each { adjusted -> builder.optional(Physical.INT64).as(Logical.timestampType(adjusted, unit)).named("${unit}_${adjusted}") } + } + def schema = builder.named('external') + File file = new File(directory, 'external.parquet') + ExampleParquetWriter.builder(new LocalOutputFile(file.toPath())).withConf(new PlainParquetConfiguration()).withType(schema).build().withCloseable { writer -> + def group = new SimpleGroupFactory(schema).newGroup().append('small', -123).append('large', -123456L) + schema.fields.drop(2).each { field -> group.append(field.name, -1L) } + writer.write(group) + } + def result = MatrixParquetReader.builder().zoneId('Europe/Stockholm').read(file) + assertEquals(-1.23G, result[0, 'small']) + assertEquals(-12.3456G, result[0, 'large']) + Logical.TimeUnit.values().each { unit -> + long nanos = unit == Logical.TimeUnit.MILLIS ? 1_000_000L : unit == Logical.TimeUnit.MICROS ? 1000L : 1L + Instant instant = Instant.EPOCH.minusNanos(nanos) + assertEquals(LocalDateTime.ofInstant(instant, ZoneOffset.UTC), result[0, "${unit}_false".toString()]) + assertEquals(LocalDateTime.ofInstant(instant, ZoneId.of('Europe/Stockholm')), result[0, "${unit}_true".toString()]) + } + } + + @Test + void explicitUnsupportedTypesAndUntrustedMetadata() { + ['optional int96 value;', 'optional fixed_len_byte_array(2) value (FLOAT16);'].eachWithIndex { declaration, index -> + def schema = MessageTypeParser.parseMessageType("message external { $declaration }") + File file = new File(directory, "unsupported${index}.parquet") + ExampleParquetWriter.builder(new LocalOutputFile(file.toPath())).withConf(new PlainParquetConfiguration()).withType(schema).build().withCloseable { writer -> + def group = new SimpleGroupFactory(schema).newGroup() + group.add('value', Binary.fromConstantByteArray(new byte[index == 0 ? 12 : 2])) + writer.write(group) + } + assertThrows(IllegalArgumentException) { MatrixParquetReader.read(file) } + } + def schema = Types.buildMessage().optional(Physical.INT32).named('value').named('unknown') + File file = new File(directory, 'unknown.parquet') + ExampleParquetWriter.builder(new LocalOutputFile(file.toPath())).withConf(new PlainParquetConfiguration()).withType(schema) + .withExtraMetaData(['matrix.columnTypes': 'does.not.Exist']).build().withCloseable { writer -> writer.write(new SimpleGroupFactory(schema).newGroup().append('value', 42)) } + assertEquals(Integer, MatrixParquetReader.read(file).type('value')) + } + + @Test + void knownButIncompatibleMetadataFallsBackToSchema() { + def schema = MessageTypeParser.parseMessageType('message external { optional binary payload; optional group record { optional int32 id; } }') + File file = new File(directory, 'incompatible-metadata.parquet') + ExampleParquetWriter.builder(new LocalOutputFile(file.toPath())).withConf(new PlainParquetConfiguration()).withType(schema) + .withExtraMetaData(['matrix.columnTypes': 'java.util.UUID,java.util.List', 'matrix.parquet.version': '2']) + .build().withCloseable { writer -> + def row = new SimpleGroupFactory(schema).newGroup() + row.add('payload', Binary.fromConstantByteArray([0, -1] as byte[])) + row.addGroup('record').append('id', 42) + writer.write(row) + } + def result = MatrixParquetReader.read(file) + assertEquals(byte[], result.type('payload')) + assertArrayEquals([0, -1] as byte[], (byte[]) result[0, 'payload']) + assertEquals(Map, result.type('record')) + assertEquals([id: 42], result[0, 'record']) + } + + @Test + void legacyListLayoutsAndNestedValidation() { + def primitive = MessageTypeParser.parseMessageType('message legacy { optional group tags (LIST) { repeated binary array (STRING); } }') + File file = new File(directory, 'legacy-list.parquet') + ExampleParquetWriter.builder(new LocalOutputFile(file.toPath())).withConf(new PlainParquetConfiguration()).withType(primitive).build().withCloseable { writer -> + def row = new SimpleGroupFactory(primitive).newGroup() + row.addGroup('tags').append('array', 'α').append('array', 'β') + writer.write(row) + } + assertEquals(['α', 'β'], MatrixParquetReader.read(file)[0, 'tags']) + def structs = MessageTypeParser.parseMessageType('message legacy { optional group tags (LIST) { repeated group array { optional binary text (STRING); } } }') + File structured = new File(directory, 'legacy-struct-list.parquet') + ExampleParquetWriter.builder(new LocalOutputFile(structured.toPath())).withConf(new PlainParquetConfiguration()).withType(structs).build().withCloseable { writer -> + def row = new SimpleGroupFactory(structs).newGroup() + row.addGroup('tags').addGroup('array').append('text', 'α') + writer.write(row) + } + assertEquals([[text: 'α']], MatrixParquetReader.read(structured)[0, 'tags']) + [Matrix.builder().columns(value: [[[1], 'mixed']]).build(), + Matrix.builder().columns(value: [[(null): 'missing key']]).build()].each { invalid -> + File target = new File(directory, 'preserved.parquet') + target.text = 'preserve' + assertThrows(IllegalArgumentException) { MatrixParquetWriter.write(invalid, target) } + assertEquals('preserve', target.text) + } + } + + @Test + void utf8OrderingAndLegacyBinaryPredicatesMatchPushdown() { + def values = ['a', '\uE000', '\uD800\uDC00'] + File file = new File(directory, 'unicode.parquet') + MatrixParquetWriter.builder(Matrix.builder().columns(text: values).build()) + .pageSize(64).dictionaryEncoding(false).write(file) + [true, false].each { indexed -> + def result = MatrixParquetReader.builder().useColumnIndexFilter(indexed) + .filter(Predicates.lt('text', '\uD800\uDC00')).read(file) + assertEquals(['a', '\uE000'], result.column('text')) + } + File legacy = new File('src/test/resources/optimized/matrix-0.7-strings.parquet') + def selected = MatrixParquetReader.builder().filter(Predicates.eq('text', 'héllo'.getBytes('UTF-8'))).read(legacy) + assertEquals(['héllo'], selected.column('text')) + } + + @Test + void logicalPredicatesAndPrecisionValidation() { + def matrix = Matrix.builder().columns(price: [1.25G, 2.75G, null], + instant: [Instant.EPOCH, Instant.EPOCH.plusSeconds(1), null], + time: [LocalTime.NOON, LocalTime.NOON.plusNanos(123456789), null]).build() + File file = new File(directory, 'predicates.parquet') + MatrixParquetWriter.write(matrix, file) + [Predicates.ge('price', 2.00G), Predicates.ge('instant', Instant.EPOCH.plusSeconds(1)), + Predicates.gt('time', LocalTime.NOON), Predicates.isNull('price')].each { filter -> + def result = MatrixParquetReader.builder().columns(['price']).filter(filter).read(file) + assertEquals(1, result.rowCount()) + } + assertThrows(ArithmeticException) { MatrixParquetReader.builder().filter(Predicates.eq('price', 1.251G)).read(file) } + assertThrows(IllegalArgumentException) { MatrixParquetReader.builder().filter(Predicates.eq('instant', Instant.EPOCH.plusNanos(1))).read(file) } + } +} diff --git a/matrix-parquet/src/test/python/generate_benchmarks.py b/matrix-parquet/src/test/python/generate_benchmarks.py new file mode 100644 index 000000000..086b799f5 --- /dev/null +++ b/matrix-parquet/src/test/python/generate_benchmarks.py @@ -0,0 +1,19 @@ +"""Optional million-row layout benchmark inputs; never checked into Git.""" +from pathlib import Path +import pyarrow as pa +import pyarrow.parquet as pq + +assert pa.__version__ == '21.0.0' +out = Path(__file__).parents[3] / 'build/layout-benchmarks' +out.mkdir(parents=True, exist_ok=True) +rows = 1_000_000 +table = pa.table({'id': pa.array(range(rows), type=pa.int64()), **{ + f'text{c}': [f'café-{c}-{i % 1000}-' + 'abcd' * 32 for i in range(rows)] + for c in range(6)}}) +for name, cdc, index in [('ordinary', False, False), ('indexed', False, True), ('cdc-indexed', True, True)]: + target = out / f'{name}.parquet' + pq.write_table(table, target, compression='snappy', row_group_size=250_000, + data_page_size=64 * 1024, write_page_index=index, + use_content_defined_chunking=cdc) + assert pq.read_table(target).equals(table) + print(f'{name}: rows={rows}, bytes={target.stat().st_size}') diff --git a/matrix-parquet/src/test/python/generate_hub_urls.py b/matrix-parquet/src/test/python/generate_hub_urls.py new file mode 100644 index 000000000..e2b8b5d6b --- /dev/null +++ b/matrix-parquet/src/test/python/generate_hub_urls.py @@ -0,0 +1,25 @@ +"""Regenerate offline reference URL cases with huggingface_hub==2.2.0 (Python >=3.10).""" +from pathlib import Path +import huggingface_hub +from huggingface_hub import parse_hf_uri + +assert huggingface_hub.__version__ == '2.2.0' +cases = [ + 'hf://datasets/owner/repo/train.parquet', + 'hf://datasets/owner/repo@v1/train.parquet', + 'hf://datasets/owner/repo@feature%2Fbranch/train.parquet', + 'hf://datasets/owner/repo@refs/convert/parquet/train.parquet', + 'hf://datasets/owner/repo@refs%2Fconvert%2Fparquet/train.parquet', + 'hf://datasets/owner/repo@refs/pr/10/train.parquet', + 'hf://datasets/owner/repo@refs/convert/parquet-v2/train.parquet', + 'hf://datasets/owner/repo/sub/file@1.parquet', + 'hf://datasets/owner/repo@v%2B1/café x.parquet', + 'hf://datasets/owner/repo/file%20x.parquet', + 'hf://datasets/owner/repo@v*/file*.parquet', + 'hf://datasets/owner/repo@feature%252Fbranch/a%2Fb.parquet', +] +output = Path(__file__).parents[1] / 'resources/optimized/hub-urls.tsv' +output.write_text('# huggingface_hub=2.2.0\n' + ''.join( + uri + '\t' + parse_hf_uri(uri).to_url().replace('/blob/', '/resolve/', 1) + '\n' + for uri in cases), encoding='utf-8') +print(f'Generated {len(cases)} offline Hub client reference cases') diff --git a/matrix-parquet/src/test/python/generate_optimized.py b/matrix-parquet/src/test/python/generate_optimized.py new file mode 100644 index 000000000..6d1eb435a --- /dev/null +++ b/matrix-parquet/src/test/python/generate_optimized.py @@ -0,0 +1,63 @@ +"""Reproduce fixtures: python -m pip install pyarrow==21.0.0; python generate_optimized.py [output].""" +import hashlib +import json +import sys +from datetime import datetime, date +from decimal import Decimal +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + +assert pa.__version__ == '21.0.0', pa.__version__ +out = Path(sys.argv[1]) if len(sys.argv) > 1 else Path(__file__).parents[1] / 'resources/optimized' +out.mkdir(parents=True, exist_ok=True) +rows = 4096 +table = pa.table({ + 'id': pa.array(range(rows), type=pa.int64()), + 'text': [None if i % 17 == 0 else f'café-{i % 31}-' + 'abcd' * 32 for i in range(rows)], + 'payload': pa.array([bytes([i % 256, 0, 255]) for i in range(rows)], type=pa.binary()), + 'price': pa.array([Decimal(i).scaleb(-2) for i in range(rows)], type=pa.decimal128(12, 2)), + 'local': pa.array([datetime(2024, 3, 31, 1, 30)] * rows, type=pa.timestamp('us')), + 'date': pa.array([date(2024, 1, 1)] * rows), + 'nulls': pa.array([None] * rows, type=pa.string()), + 'media': pa.array([None if i % 11 == 0 else {'bytes': bytes([i % 256, 0, 255]), 'path': f'image-{i % 31}.jpg'} for i in range(rows)], type=pa.struct([('bytes', pa.binary()), ('path', pa.string())])), + 'tags': pa.array([None if i % 19 == 0 else ([] if i % 7 == 0 else ['α', None, 'β']) for i in range(rows)], type=pa.list_(pa.string())), +}) +manifest = {'pyarrow': pa.__version__, 'rows': rows, 'expected': {'first_id': 0, 'last_id': rows - 1, 'price_123': '1.23', 'local': '2024-03-31T01:30:00', 'payload_0_hex': '0000ff', 'tags_1': ['α', None, 'β']}, 'files': {}} +# Eight-case binary orthogonal array: each pair of CDC/index/codec/dictionary/page factors +# covers all four combinations, rather than requiring the 32-case Cartesian product. +for cdc in (False, True): + for index in (False, True): + for alternate in (False, True): + suffix = '-alternate' if alternate else '' + name = f'cdc-{int(cdc)}-index-{int(index)}{suffix}.parquet' + options = dict(compression='zstd' if cdc ^ alternate else 'snappy', + use_dictionary=['text', 'tags.list.element'] if index ^ alternate else False, + data_page_version='2.0' if cdc ^ index ^ alternate else '1.0', + write_page_index=index, row_group_size=2048, + data_page_size=4096, write_batch_size=128, dictionary_pagesize_limit=1024, + use_content_defined_chunking=dict(min_chunk_size=4096, max_chunk_size=16384, norm_level=0) if cdc else False) + pq.write_table(table, out / name, **options) + assert pq.read_table(out / name).equals(table) + assert (out / name).stat().st_size <= 1024 * 1024 + manifest['files'][name] = {'options': options, 'sha256': hashlib.sha256((out / name).read_bytes()).hexdigest(), 'schema': str(table.schema)} +# A synthetic legacy [B footer exercises contextual rejection; the original writer cannot +# produce a valid byte[] file in this checkout (InvalidSchemaException, captured separately). +legacy_binary = pa.table({'payload': pa.array([b'[B@legacy-text'], type=pa.binary())}) +legacy_binary = legacy_binary.replace_schema_metadata({b'matrix.columnTypes': b'[B'}) +pq.write_table(legacy_binary, out / 'legacy-binary-metadata.parquet') +manifest['files']['legacy-binary-metadata.parquet'] = { + 'provenance': 'synthetic legacy [B metadata, not claimed as original recoverable bytes', + 'sha256': hashlib.sha256((out / 'legacy-binary-metadata.parquet').read_bytes()).hexdigest(), + 'expected': 'unrecoverable-legacy-binary error'} +for name in ('matrix-0.7-strings.parquet', 'matrix-0.7-date-units.parquet', 'matrix-0.7-metadata.parquet'): + if (out / name).is_file(): + manifest['files'][name] = { + 'provenance': 'original writer at b2095159664b34ce426685c387adf0bccdb33e01', + 'sha256': hashlib.sha256((out / name).read_bytes()).hexdigest(), + 'schema': str(pq.read_schema(out / name))} +if '--default-cdc' in sys.argv: + pq.write_table(pa.concat_tables([table] * 4), out / 'default-cdc.parquet', use_content_defined_chunking=True, write_page_index=True, use_dictionary=False) +assert sum(p.stat().st_size for p in out.glob('*.parquet')) <= 4 * 1024 * 1024 +(out / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n') diff --git a/matrix-parquet/src/test/python/verify_matrix_output.py b/matrix-parquet/src/test/python/verify_matrix_output.py new file mode 100644 index 000000000..ca6c706be --- /dev/null +++ b/matrix-parquet/src/test/python/verify_matrix_output.py @@ -0,0 +1,21 @@ +"""Run after :matrix-parquet:test, then rerun ParquetInteropTest to read CDC rewrites in Java.""" +from decimal import Decimal +from pathlib import Path +import pyarrow as pa +import pyarrow.parquet as pq + +assert pa.__version__ == '21.0.0' +root = Path(__file__).parents[3] / 'build/test-interop' +for codec in ('SNAPPY', 'ZSTD'): + table = pq.read_table(root / f'{codec}.parquet') + assert pa.types.is_string(table.schema.field('text').type) + assert pa.types.is_binary(table.schema.field('payload').type) + assert table['text'].to_pylist() == ['héllo', None] + assert table['payload'].to_pylist() == [bytes([0, 255]), b''] + assert table['price'].to_pylist() == [Decimal('1.25'), Decimal('-2.50')] + assert table['tags'].to_pylist() == [['α', None], []] + assert table.schema.metadata[b'matrix.parquet.version'] == b'2' + output = root / f'{codec}.cdc.parquet' + pq.write_table(table, output, compression=codec.lower(), use_content_defined_chunking=True, write_page_index=True) + assert pq.read_table(output).equals(table, check_metadata=True) + print(f'{codec}: annotated strings, binary and exact decimals verified; CDC rewrite preserves metadata') diff --git a/matrix-parquet/src/test/resources/optimized/.gitignore b/matrix-parquet/src/test/resources/optimized/.gitignore new file mode 100644 index 000000000..0b7029b7e --- /dev/null +++ b/matrix-parquet/src/test/resources/optimized/.gitignore @@ -0,0 +1,2 @@ +# Original Hadoop writer capture creates local checksum sidecars. +.*.crc diff --git a/matrix-parquet/src/test/resources/optimized/README.md b/matrix-parquet/src/test/resources/optimized/README.md new file mode 100644 index 000000000..3ec439776 --- /dev/null +++ b/matrix-parquet/src/test/resources/optimized/README.md @@ -0,0 +1,42 @@ +# Interoperability fixture provenance + +The manifest records Arrow 21.0.0 schemas, write options, expected values and SHA-256 hashes. +Eight cases cover CDC/index/codec/dictionary/page-version pairs, with two row groups, raw media +bytes, local timestamps, exact decimals, nulls and nested values. CDC uses 4–16 KiB chunks to +keep checked-in files below 1 MiB each and 4 MiB in total. Java tests inspect actual page indexes. + +Recreate the Arrow cases from the repository root: + +```bash +python3 -m venv /tmp/matrix-parquet-fixtures +/tmp/matrix-parquet-fixtures/bin/python -m pip install pyarrow==21.0.0 +/tmp/matrix-parquet-fixtures/bin/python matrix-parquet/src/test/python/generate_optimized.py +PARQUET_PYTHON=/tmp/matrix-parquet-fixtures/bin/python \ + ./gradlew :matrix-parquet:test -PrunSlowTests=true +``` + +The slow command also generates a default-size CDC file under build/default-cdc. Python is not +required for ordinary tests, which consume the checked-in fixtures. CI installs pinned Python +dependencies and regenerates both kinds. + +`matrix-0.7-strings.parquet`, `matrix-0.7-metadata.parquet` and +`matrix-0.7-date-units.parquet` were captured using the original writer at +b2095159664b34ce426685c387adf0bccdb33e01. The opt-in `ParquetLegacyCaptureTest` compiles that +writer and its value helper together in an isolated loader. It records the mixed Date/Timestamp +1000-fold unit corruption. The original writer cannot produce a valid byte[] file here: it fails +with InvalidSchemaException. `legacy-binary-metadata.parquet` is explicitly synthetic old `[B` +metadata with a textual payload; it tests rejection, not recovery of original bytes. + +`hub-urls.tsv` comes from offline parsing and URL generation by huggingface_hub **2.2.0**: + +```bash +python3 -m venv /tmp/matrix-hf-fixtures +/tmp/matrix-hf-fixtures/bin/python -m pip install huggingface_hub==2.2.0 +/tmp/matrix-hf-fixtures/bin/python matrix-parquet/src/test/python/generate_hub_urls.py +``` + +The reviewed v1.0.0 parser uses resolve_path; 2.2.0 uses parse_hf_uri. The later convert-ref +regex accepts dots/dashes (`[\w.-]+`), and revision separation occurs immediately after the +namespaced repository, so deep-path `@` is literal. Special convert/pr refs remain unescaped; +other revision slashes are encoded. Revision text is unquoted once; paths are literal and +quoted, including existing percent signs. The Java parser follows the tested 2.2.0 behavior. diff --git a/matrix-parquet/src/test/resources/optimized/cdc-0-index-0-alternate.parquet b/matrix-parquet/src/test/resources/optimized/cdc-0-index-0-alternate.parquet new file mode 100644 index 0000000000000000000000000000000000000000..917c07feafd4fd5733916021c3900bb64a7744e9 GIT binary patch literal 51586 zcmeI52SC$W|M$~UpbVugl;ME3Y*E>Y$Y@KEAzKBUY;k~!fPjN7w1~15Q2`MZ5fw!7 zUKgMRQBhndiUV;^mI?^Mdy+!q0`GXn|9vi1+Tc9CyVdLK*&cH&-;ZSX-MrSpC*NBNzs)g?2(`-lkrjbi9^yS>sSq>LvBPiKI)ayZllwDreY`E-G%Gm|8>KC{C(T z*zh*x!ua4?;)SvPMJd&CK8eI?nT38SRpdD~#41VKi78y7xj2z4Vf;4v{3!ie!g;Yt zMakzxloJW(gva|OR|=AC2$ce(CMH*41jPvzT|u=4nE=Xt&v`p&Ly z!N=z~yUfv_GuvMO_-rQ&vi@ub(~rky*-fq2n`NVU{FsxqYKESZr9$vA#~Ctn^c-0d z`o|oY!el)MdjE$~dy~#vy7opb$4c!CU!?2W89WLqwViTjwyv#SonEPp&IKu58?AC2 zGaBlCyhk``yy5Dm-#qpRZI;vVtO$LaC%B0`!710kBM-+R(wLE5O}ho+M~%^O|IK|j zcB2T1Mh|Vu#l#4Tk|+l5xlj}ai-o#+PPm14)=za?-*WuKqO~tFrY;J76ntXgnmco* zE)1&EKe1r-1@hDdtI9u?&0le%e#(5mBgf0;E!&?lW!{pU;Ig@kx6PR{*E>nS%+)iV zJjK-=)9+>)-t*zv`H;Fr#%nvCR>gE)T{wNsn_GL$Lar{Dw)zQoqd@1C`BPUk9`iZB z=E^+%WjFK9V?Nc+)m?l!UD<3+t*f@@`B*`LPnTUZ7o7@vb}sm`^Q3vD-c`{bFU`?# zJ~(?XGx*YMb*DY7jo6PDXQ|rl(DOMLbkRxKI!(zu`a_MQ;*3pF%FLh|2P!iLCy4!U z!Ct{+-G`@@ffww?8wNh*Ms-x%j-9gnRz4%J+D1;tiyMpSsG2D=dHyl)%GFiY(#7K!o}pAHfinVol$zd(?ThWy7PMvmreXJI}Q6R?l>9 zXc<_20dM22E*#tytbQ8Y^-<55(83u&+w?*8|Ak*l=z3lSdyW5D1ry*!2`U(tw}e!% zA40)4w1Sxvp?nJ1AA=RZ2?`10MMTBKM~#*s5J{3!WN8^$xiMqMjh9!TP-%)v$||aA z>Jv03PMWN#rLCi@r$1$?!8F6^M#d(lbOzImWj@2g(#m?Kjjf%%gQL@|*`QXq&Yd@Z z!NNsu?jD|A-adbj+|Pf-%2ffY1A~IsgoK8zUAKNictm7WbWH5VxOg@vVN>Gf zElJ5KscGpMTeoFy&)TtbS9VVB?z}zud-v@xI8b=7=+NOK#U)2ej~y>N@!RhwPn|w< zw!EV9+<9(Q^@W;?moC>{xmtJa`i+~n>Tlm^xO=bh{)2~)njSYldHU@6iFb9C{^1!gpb}X0t$~Q>@up6xPET=C%2rnF@%pRgV1|>P zAYXd8ucY@w=uf1F`^QU9@E6(#K69sIf7;? zCjCyM#5Cu10k?OA5V(}*x%tw|--pRQ%m36|G_?ge%xco@`yI$(b5}lO5J354v$5nX z99lpSnGW}nMhfWXdISYT7_xZaV#D%kWea`0Fx1W>NEa*n*LoaCw>um)<5u$7qw4~n zm7HgVs2-kV>$k|ZcW&vIqA>k7-#IUxB_o`@PiJ*$6h5zKMd8dAK-lvX-C0;O$~Cji zuf!%$3YLHI6)B`T(Ap))U@Zo$!L_vfxtmvji4WeIaEq!IM;D1U#8nYCkv5q+g(sS1GFCd~$ihz3z0X8p9zzR~D;`e@&1wbBNBJ95o|gLjpcQSTpRFGhw0AkeFGcxcHvqoCBSosx@kJ|P*do~wPr>pNxciI;i zuda-_P@0!=v!<3^bFGOm%MCk4BpZy_tR7a-~3y)_LiTK?G zWAP2IBuV6%2YUVGX;>VKi+{?F&pT`Cs@fBK;FT(eBa#877-(xdo(r8EERY|MCt|c| zSS-uGIjTcaRVA7|E(&kK!dXm=vY*mt>{D*+6=Cey^&z>ZBeuJp)pKFb)ANcQ1cD-k zGJ{1A%3CYp$)sJ6WqES!SvxZ=CsI=^nOGG}ROhD8@x~Ol&vSsw)|2q%`2iQl=zCH}+#d&^aCsg$^n zBpqUqXHLZLA1=(Iwm;tfl1QY|2s8>-S>CFXph6-^Dw9-5!89h#k;%+ciA+AyQRgfO z)NvZvqK7j&IHDNlLQu0HNv1-1eE;XM*Ah-#$M?H=r>>s4 zoz_3+0md{iUdkhaW2*RUEA@0;*}Ed)o-%*k9bLLkvo8_)5~Y6k>cxn95{d=|GPA(r z{zFp%Adq@scLcS8Ks0d*YaklU@1qi_JD5$VJ#1#P;K|XtM;#OjjbKipEK-)Y?ji1k zxUt+=5xb`Ie)5$f7CV!ZErUcNey2ycskgm&v&(`-Po%Rm=|-+Q!p|vPQ?ygERJh@L!tu~CJ95SLXra_shBrrMQp#O zlePL@n?GMd5sqzB>E-hH-&ZC* zaI>G1ll`>MXid4%-L6iuQpDM#s^AWW$FT^+YfAEj(`-qOq_UFbP6C5oRI$F-1s-}T z16HPK#L5g>kzf4+1&tfV%19sr3TM~~K{qxMhZ2x^T6N$i@)>(0jWH+G(q`mb@o5Zc zgj+c+l82Vu&$DTsi(xY;-ZWVp&NH#iy1;QzI`Smf%dm7-3gyUeR#D1!kkEGYHq|b1 z(4=Zh>SvUHh!=iGr2*Rk5s8>J&6peOxMuisilaR|n@JmAon?to9&5I*OyZVjcLAlC z)&;p`clEV#yW@S|Qzl);~m=0o~9g&>IoJ2E3nEFvv$D=)~yU()S&QNU2KpvBv)*5+_tI3+XLAVp1E zJ3xe}Od=^M8HwQCKPBy#?R^~fiJ(ZJf>v%QDXFW6O^{%U*3S1Q15*Nbo<5A2l0g;x z>KCYF%rK@z0TIc1#x5Z!l`An|N?M^bvo!~gSc1-zj`gRP-mzHLQI#)X! z0(!A^vps$CqOX2i(d6_8{EZa!QF0$RN2=m6m{v^{UDMQPkxV{_R_dU`svLw9FR*E=>7DK7?oei zWNjDS3%mlpaNA;^i}c4>u9+yo8W{B)g*=>~8EH|SGp=%>{)%kINJoWys} zZs@hWea2gnY~iilaH!O!GsMpcR#LR2yu?X!3j*L?5IlARIze#XTI=~q0YyBSAhbq= z)b#X7cUmhEPrwri#LJ#-!jqak-YO|M;LZBV*W++BCXR+>lJGB+3Ruxx7EOg5SQMYvmX) zF-J$nX9hL#t6!iQ*n-MaJl+{YiBwiuDdKEuED@6gP|Eb zm2+m^!COn`e?GDBrYYz^zf!)i^3=Zm)XsZNfDa%ERjq=d}f z6B$^~0*@MA4+(UkU4@B2J@gL=0$J@6Z3F@lv}hMoXEZZgh0l45-B_J zBI~o(i*RO$GjIeI6(xDg4;pW&ZzEMyY?SPRG`Ts-ns_1`3LtVwniy5ZMYD~$P9M3P zu!l{(m)GN2Y&H;MMoq1SO|s?AOoJ4U&(0URuLFCjy-zWr;W{_lfqs3C*@q$|ms2hAjIfu%HFvv`_hh@p znXFLrPObd@{)jIn{*9*=*&1oIYvpr%%{v!J9)ip%F-Eu;NS(2@2ibTcl_ZhXg%=@lfJ#I-DeTPT%uM$xCcCKH_8=CB zjY=_96RPz=BN1!9pN#`HuWT8U(UnT2z)64KZ93OGm+cbOU76`KV&Zk)m~nRhXJ zZ)b&m0xexygjGw=*@y_xsHrhK0EDu7cYPBSmoMUof1ao<&!iyTY7=s5L_q;I`sCq$ zwj~(HRVU7;#xZNwQbk0#UnEfgAE!daSt^sBL}s^ z_QVuV5o^*dWv7V@(4b<0If#^80tjOWdXCb{>4Rq$;IZD@-^@*ojMV&Mq^4(R0SSa7 z`e~<+$3x8)Wa)w7yvVWqks7bUBkz4ba!d2tZt8`@V=@~P$(v{H1oH?;w>$BjPMF^v z?p?Uo{be285l^-<($LAJ2A}TzLbujpPkCy;7PkxnQZ!mnn0%~vLo@kIa|Ue7zWTBn{kqg;KpTRnV#vBXlmI( z0OL3W28C8=Z1a_6h<4|C#QZ4)-`3qATt zBI&jTD>5cU3TTfd-hnI)${2y3a$(7?CmjtG< zvb+56eCSopwz5B+ml95h_BB^jPVhAk%=XO=byoU9!PQy2;}W6_A)L6V86j4DF|Tz5 zI_-tiKP1poSqdVa6lG`hMpQdFlVc}Vj2Q4KQ;)Gu`#*I(raKSz5f>@iy zGYQbTj&x;tiX=y-VeH??)0`JJWh`fF+oEJPqg5~XRMFKiEuQzHd8oF|S3#vO~e=gmx*(Xrg?M){ab`@0i>;esdC3r7ss zukLq)+u=sim*J#^7_K(DzcA!kMfhrj@Kn7^ema^jFX}9_kloh|q$M z^n?47r_+U?K7FYdv*L1bDe%O4I>iZ0pfep)LvtM#8a$w;0nBfZF9uug%K?59b2EQVmBEMCh zEA40U#T_UqZU*_eD46XOg`Z%DzmO-`r$A}~{ZPLwcv7+-WL~5wuj)k-^vRHtI`8?( ze)#7f{!G=#_5W8NrG^jC{gn1d)%qtt zrO(qKu)+UQX#;aUVzv86toHw9t38_h5Q{v11sWvq!+zg3NPuyM8zk6)1__od2q}rc zn*KjwwfVJnLq4IkN3Q9P0HhV?1kcvSS&|z_;IhQwb{71_ds7g3j33)Ci$2c(T?l+- z;E8AC4ME};C1{KTT|$h|^;W3Aznr0SQR+T43ge8~c|Tu8mveFEyrZeJqnjAYWpCTYqWBR!yY5n~@@*V68)9;mR3EH`5+!OqTsV7S|`z_cr z<|eyZH>)Jkd)yw`^Z07bwI!Pt+|QFPWmjp;D@mBWGf!#{zDm`wgk!THkCevdDvl{( zv&Q8SV(?sr&&BaZ_jivDWS<{9m`8%9V&UXowJzJ%v?iHKiRecHro!Q) zdS$yNwa#kVRv{7f2xBS`F4&9j8r3S z+l7>+?RqoY*YJ9nUEtQVolm5E4Jx~BSFTCh(Rgd|l#(#p<<4nYH`5pE?e4bmoszcw z^6bSrDPcBV(rKCJr50;NbuhrbHpi}M@Ufr9}P3#!^T(wp+Q$efTE&Ocn+U~aY#%KF4Yi?7RTs}X% ztan-W_4XalR4;38RhU@r5`MUMUUy0R{Ac?wX=W%)D4!L+uh+UerG4x()k~V`3Tox{ z;n}^0-2v^5PxoKcOjA%PpBcWjcS5&g`;Moo7d2BAl*%o_xAczb)^4BwbbpOziUO_N zEIh7Pw42yI_Ni)(X0igM+%!DA_j6ZUTjP`c7c`R;X85AslU)IAjm`V3G&d>3-I6{zXV#UACLC>Dus)-lyGF9WgIDuh#8yn{K)${7vtz?!6sm zFG8-??R1-Fx;p$xFSmPRhro-@E1J{7r^fB?$iy8u@G-$)`~~TXUQd_kSNcIV z*S0Nl5;Z=O9_#nYJh3sz2mgU|MC{&HWsS(r8}nGlDdwYimhQFi5SC7Gz|sl8??WS&4h{Ln(wR#@ zGqhI^7&=pJ&Rn@hXf>n*l`XrTrK}-cucK;RY&ify5e5( z+&wCzvUC?;N|sYp5zo|IbaK_*JY})18qS3d+iA+8>8f^FT66Q1L{b%JBoO3iO88_2 zleMi4yA_4EjGeOVS|(LdC{bqeykghgG{FQ(73*YKDhpCYkXn9md zerd1YbZ7Fy-&Z{l{a*10hfhIap-|5!kkXrfwyt2JdqMZC?&$b-mofn?IBU+k(G7+L z`}^#t=lR1=)y8R&3x$AWzI%-x5s2`BWQ6#VK|^5448A@uW-boRpw1h3eZJgvmQXsh z6&w@#zTtqG@0Ohgs~-Ng^!(z@S$a#ZW~!SlPR!JDKO6e$oX@7M6I~Cx9$@+;q^mh( z8!SEN%}G_V+#;*a^kyeh=;2*2E4|{kj5Ao#P{8ntOO(}G^xKk3&y5LEYId1w49{3L zfnpN+vce-KPF!}fYXRLOIu=imH&{~P9u9fJYQM|C|3soxXB5B z@tT}CF7jbeThSy#$5I|nTNW^?IGAUe8IH9mGfrS0vSP(2I9C*J=nj|O# zaVH|ZbzZN6-)BRa1t_urB^IE-0+d&P;tEh&0SYTXSp_Jn03{WmpaPUrfMNplk*d&47{_P%s0^Wk9hED3t-83*bQ+P$mP4 zWI%}wD3Af=F`zgGl*WL<7*G}iief-X3@C^J&D195Gg+ycapYs_28GbLhAy3MWC?Bq*8$C6l0F5|m4V zVo6Xc2?`}anItHZ1SOK7KoXQkg5pR}8VL#`L0KdyiUcK*pdb>ILo!&=@ggXN1%Fl@f)ZFz01L`rLGdet6@9rM3SU9lD=2yeC9k016_mSzVpmY=3JP68nJXxA1tqSa zz!jLc!n0^7ZU&{zps*Q~HG`sNP|^$vnn5`;C}sww%%G4NboK&8%%FrB6flGGWl+2f zN|!<5GALUHMa!UM83Zgdd_|AbCuhPFjd=1RUPC_<4DNyVz!!aUKNlMH2bwD1B4 zcVfGccfu`IA@U6P3nJgbAflb2Cbrby!`Elhw8_g>`N|HgXu&H$CjYUvDDUOkc~bRw zh_-2*bguGJ0g7S7I$Wez>J*nxE0)cXiaIUG&6T~k-RN^sL2CM$+`0tAeTL~b?4Nan z9;+*|Ox=?G_PVO#D+|4QA!`;Z8-E6y+h6V({p`B0)2r*g_1YO0IdkIMUzX+_AjEjq z?5Rs&Gqh(D=#V%bAFYLrEGTdfx_K6JZPs3GQm#9GHa==npk=Ra6jxCxO=6==HttF` z?xj@xgnnYN*qb-Ox@M}ITm_{bXit3;5ZKaQT(lHpe6_pW!ZI;Aq($-K1RG|Cd(489 z`er9LT~|vgT68zlMExv%U(kBtz_^=v8nU`Vk_`efiZkbjOunXPzt~n(^VadjRq-$H zADI}YrQ_|YyM%e0?A7$_!8WhVX>-#L-c@%_g&z~mav|Rm{PV?_SNNZ?M!bfAm(+1# zXtkxZRIaK&w9&t+|8mBHyJu!Cf!@oGQDUu`o_3*>=5ph%ST^~0uEPcOFAgS27p5Pi z&HNmiW}GdO(npkNsl8NaIl(wPWv2MEw#54Z@^ys*YRIjsJ>+na^m~Szu6nW;yk0tY0lkFDVJ$|8P5-~p`ZPn}M%b^eI!Y|?uFeJAm zjekgg|KR<*<@5GVns&WRjU_JEA(-)Hx#!fItyhe7>lfx8j;^{WHs+4%v#~y_BbVd} zE6pfNp3D|M&kB2~t5UXbiUPW_trF7#jZ-60j>d`ml{>y1Nm zom21KeWZvoZqA`rUyam@mn@m5Klp0A&b*X!sKAWla$xp^4C}VCye$W~r$AZFh-6T; z2>9sa$bAKm;RmdX_uL2~MJD6@;IeXxjfm4~v!Pc$6&RH={fJUloA3?6*xHmOGp;Mt zOmc6(1O6J=l4 z!qMWpPZ5@l6Oi21y*QE$kXRO=*M(B9Kb?(8$$m7pn3tR8j(Sn>p zE?z;~bAJ0%OlJdbr?KLukhEmaC zP9DE>MITPk&)jgV_`0(y_vi{1`uL&AqPcmj-99vfq_GB&*tju+6 zESr?kUR}B!1QzClg$e4T1gYuk)bJRA7zP2-s?LneE=c}#<18q<)O>A{dnA6gc+|LU zvJ9fUb~Q&qCb52MW-x0^&%!rGQ42B~RO@GX$;h@=ysElrEM#o3Az(#Pl!V=p z`qeUzluuMnw(EL8)zWDeh0c!=5N>0zyjI1ZEPQmW)<|x*SjrmKygq`|mC8#NkCt4G zyZS=FAi%<)`{k8`8dch^TH@Y#%@@BJ2x_FS(zL#R^~1Wl1&n={6n^i>9942pif0)Z*Qgz7Tlm|1W)Fr#!G!w@2+GFn#T8*V-t@HU{LWz`cH zL=##v+vTY-bxU5wgvrKLQ6zxy_@Y_3|sIU`W z8JEqkEKIsHU&SC|dQsBNh|Z<9lSR8C#qlzC+);lNTxhi(_(|BDqF~==(4HH1gM2;s{@ZN)wrcsEnLYcX&C}hsCz4by~~5PUB8 zu$O3hjc;y)o$RdxTk1FO%UqcEihfon&D>;r#@?rC@~?%|baw^aDY1k9-a0Ja_42US z9nLFu)tDnTgaK!E89 zL03zV@1r2=POFg5Q5KMUXAKN663AQ0d7r81V*uU%w63Y(cxUtCD%Sw=IEWKM60d`-^ zOCECevh7GLrARDZNGu@TaAJ99_v2k763Y}2907LcoyHG|r5Z`z6(o7v2Qp!}9FF*P z79+6?mBSIg&f;NnIKuyzSv+hG50u||Hvz0FJ+kMg(s_cWQY%us3{?QvTU`&R#ehft;6j zmYe+3z3aKkJ1=tF^siz98@@_PDUvu|@#3}vD}$FZeQ*Je5?px%KLYq9 z{Eb)9D}M(z@ON4VXV&0ZrVlQ_hrI=PF?c+v!1v9X5L-TK)z49X5Z5&EAMVXdnK1 z{tlbHbG|ZF!`X*FlfOL9M*(jB0;Jdxf1LnzpV|A>MF4?vDbN521|0CJIk__OT;Dj62VJB;+i5aq$r z6Xh)-()%IwC(;xB$4gJ_A1=KCmHbudjp9pB{444G5c(78iT~rJH|ifQy#bZ{Rq09a zr8oL3>HQG;6X}iq$4gJ*A1=KCl?*FAgneJ}J^TLnx9oczgy25l4hEp_2cOH~qwl8y z^1U`k$3PL1&(DPPD%Qy7?}TpA0mXhS)*%(TrTjewUow)3%XR&6o3#=pj;59=e1HN4 zBVZeVdt%W5cELs8?&;`I_bR%?uq38lbgYRLox*(PE z1NojofXVmv3HqJx>41D+QkNA&5mJAin|;gkU8S1pp--MUAD`33ooX4zFfiZ2Zr-~p zlQ&lw1muAMP)0k!oB9L%J;Xl&1iMxk!QT(Qng9JSu--%YCP0d)kM}B63w^T?q*FbJ z4CWKnoc8gEYJkaj!n!Qp0I25t zHBe0uMq=r`mJybZiey7kvL&I?5fsyY?qq6|wANFF%K(u@pwMUplgb+==Bi3IO4hS! zOr|HGMPsErdKsJSEsVBht_o1C%Q=@sr?c2}iYK$u*punW%wu_al9);=BuN!Om?kPw zwi`ts*zmTv-O>qu5!){vLdUjhI zg8WE~$-1mX4(}Os7M(?jrjOehdaT0CvUYT7M5U+^K`MZ&xJ65ws?sENZWAe~Va~Q> z%VI@J!ZcBIp;QknA!73NP!PacRm5Mti6p%j4;V2!U)~x0 zvNQN{#93%P3+IVvalN@-jZY=2Plul~>52t_dzJ`EWG7uKB{Q>P2POZ`>Enn8GZ=66 z{u^U5g0~vpnEWAE1{;K+qbNuE*_aji^)K?%VX45Bn7KyVj5a&#y@x5}8^&BPKFDK8 zgSGL1X0<1PqpJm7su;~=I0#X>0=5MKZ^86)GC+{u!jKv+!XW7*d(Vy+!*iy;VH2Cw zp1YMuU?yj31>~`!+nBu%jG3{p>kJz0x!V1TXU4ditFf{4g)|X@$*JDDh;><=Z-`y5 z9J_K4i;ki(ZfIV9bVZJih^} zQuGa~YQ)LP@LCxfSufbfb8Ha7*dPq)2ulq?Li-MZR+l#o@ayN-y4j%pJQDxVf+7tM7w4C{L)uZ_hqS$HmkBRuXQkyAlrt-{eP z(^($BnQAj=D!t|{ZgrzgjGKGIxF4UXpEs3{>3F>R3!aT<15jv~WRlkxyod;iBqI6j zPI9&TqGbnxlwgAA?Gdx{ebaWRb-2m zWM4NHUkdno#x6!0C(pnX&(DD38A(eHP>5zkN>RB&palaz3OE2J=Xp_qn=qLq7)i!a zjdHN{-4^az?z|3f9n4|j*5UHzpCAzq`&9Q%4aLV2*??je&ov7TXLDnLu2>uq8@6S- zWpZY+r6pac^L*(wvqUzV4n?in78?56!cr+u*z%PAqvJ5)6-JVGjWl3}(!=-RKp#V6 z>jnFG-VN3>4k6wReqw4$=JdZMAJwZa1&!D%o1O!h>nv-l4|f-+B>BU)rVafj0401M ztHzGPn+;GF2e9>#;I1@9h~z2`_lCS*9}`GWQ6erIk7c!fj>TYs4Ko+aElj)lE;qF?!FhsQ0uTk z`L(Bm7J5B^*vAbzRQaI`?W_rRnysZ5PJn-F4Bkw(m8^Tc!pi92DONSdH)JX37=;?M< zT_1cDgrd!d@{e&4Xl2&+M|!uprJoDyCF}GKa-64T{0_T706{00F44!t(1k`a93GS<3wKUct>L_tq|yQ!jY0FE1$YWphYGCj+ujuB z+3X!J`}m-$wS%U6$NnZo;QxX~pk~rnnbNeiJ!sn0C=vm1qru&b!;uxK36`kzGbzJ_ znH+#8_2CxE%aIn!_X-$l9X2Sx_Hxic_W=ea_Gex(HVcPZ2$lKonL9Om^YPoeLNCCs zFjmZfU1J!Z%~}$?8^>zE5@ao@d6LtBQw-qP%`^k{3Q3(@&snqASxb|Q3oe9tnY7?y zn4TGZ^7faB41JvPq5gODpXVJX+$e{rW*_@@#H+d-OU_ifAW8{t|s@WM8 zkZbpkisE^jRI|z-bVsSaHFpf`gOUR~$bcPjaQ9=WLYelBBp?*HSG0AcS2UC`z7GdN z85%_|*vE5i5W!FuhHh|wK&YO71{itrgD(7FxB^K@dl@SMYf34Q5HfwL1!Ko&5Ar;+$@LPslX#JPe)oV-|J+k zb=c7S+L=K+eGc$likJRLTNeh5JAVLx@qnD_fE>_m2AyXxIEOffE-wLU2_(JmcOopD^?wLp@@A-PZLt7xSVs7wlzGUJx2{MPny zjzm%l8%zSII%a;hN{=n4d}^`kYdrtyj^r+Ti4N1cGj7?hS}vG9Q;CG&5uvQ)7vvg!g_@^^|!^muCioIappc@1*Bn&)(eMN(Y9S&Za zwAbeP!CQARPq#=OalL*sp*3MTfY-0yL~Sg4cgMf+)0ZBuhVu&T^w26%;A@a7SxKP~ zyb^^ScmsVfR9sGg)PQ&b)673RURi}?Ol5lJaaqwgy5N&h$NFYH>yG6n8+)=2Pu8xS zgW1u4*8)r+0lygrx^0wF`<+@T;Gd&pSMf!A;KNogWtC)K!e!06&23MtY!tFjN)U-r zcq}nRu`Sd$$-o1&nkr|S%{vly;?Q8r=j4b%8cG@8hr6s_Kb=QTHD6u#n&UgCMqoD!b}hIDaiH8HSyz}moxrRh)*47uQrx!?nlTnUba<@> zaNT98-4Frac}i|aAigUO@j&9mu*WcM^*;UnWul13Y~2@*_Cl zSz`!vxRSgY?3m-Xbn5Nmj#qB5qKZKi>Do6ELe10kaK#$T*Hp+3z;;jrswIzBa(Oz{y~iI|?7oa969ARbEmM z0FV2@BRloLA>mATWM{$qdW?MKSrbR*^s1n~FjFiOhr{Dv%(ykG$Z$g*hv77#|D-|` z0fWWGj^&=u>yLTx-rki}ivv^GcIg#3ya@0dX07Gg8}+hp!^STiKJ7HZ3_9?NLEJsN z+2adG8TXh-X3&(FG`m=p)|8%3?%T$+FK>Q-chjt+PWR1a$({?kJ@FS_DuM}6wgHyv zSrC>h+`i2f0mF^Z>8nuY$Tbu?(uh(=o}<{2Hk3Qki-JdlQSt~0MURX}*&}Kwd}QFs zCX^?Gf*eqE6iQkFLPuE+C_M^=t;C?Tl`SZ4Wjo4S$wz@JM^NI*Nff#A|L^tl$Py!T zdKBO@1ts_}P=t>S%J6X+krqlx`9EB>_|TLQI{p8e)j}C@C~YVW#SLYnyrBaqaOfCH z94beVLzhtIP(2DAYC@?)Z&2*eCzLz%-+fm37gqa!mQFt;vsrWiz>DVtc+n8hKSJsB z-!hw_i*El@%;p~7Tqp{I{bw_qwZ39DTat&A%%}m$i18(Zh72v4oOMUz#s60@-|qb+ zih=uxWDH;K`$L#-i2=!s=1T?*fhF^E%(t5xDmVd3yAt~Jdb`tu(j?4|-cmLN#_Djt7eS2~j5A&TcfcehjVZIZ-#eDy- z@V)tDNCYQ0o-|4Zx`i;lQMM=!hKr(H05TtmOW`ANHTXShv_p;dr_t^-+M7nlY5e&} z+)(}~4LV4Jj?th)H0THo+SNhFXVBppbaVydcm5Re{eOYwKR}oqpz!|$^F2c0 zmsj97IVSL%9Qx=OK01Vtj^LvM_~_ISzsV88Z*oMV^Frvf5IQS_P70xOLgG8R}9d;y$a3L zzsxpCvgoNzPWR)InKP4Qu$Xr-6$f-m%{cyfjw$d7r1Jr1ts7i-x8+XdT$1j}Gdo?)z-W|UQ zb7mC3Y3N?>vT=)Kqu7bPJ@vlX5~C0g-f#&zk*E43~! z(Zq@_t!CT3dAMkI_p=4rFKbqJKggY|I;XofKr243)N`ll?IvujMR8Aq=AOHWPnJGk z3cos*=t55a^BC#>SZk3-&RrmN9AKm`$_-+qVcx#FuK8EI{q<{{XIourZaYyIqEgie znEIra(aH~1t9twQ51MtQ%|F6@e}cz3fU92mTcNDmOS6RN!nhf__`g%&OI!* zs`SKErnF){K_^t(>Fnz@XNvCLEZq4U`Pi-AnW3)NDg-9#|2{|jMF{TgT6@)dtDiy_ z#Qf@B{XVt2wmL@`7qj7J@Cn!Yt{1cDvGaUCG7PCYIwa59ByO|T^BJBVvToO&LK2RI z50C6q4>ZEq;&d1C=RXgW7XHt&_T8my@z9O$NZ_lV?cr>8 zp>pzJzyY_Ednr1~EEY5BF+Eai)vlAH7F~;#6YvmADf52#nS9K5txx^RCHuD2)s$*c z;*EZFyJj|Ri^+3{5^~Tj}Iw-fMCE+&xbMK4*&-I;URzle*j~^A0DcC z0x00#m>8xzB~e=`GNwTLTM9TnYRqjO1e~i;xh`d=d6e*i2~)dUWgTU1OncQmwo%4I z)Mm=ztRBxvk6y?u@#6-It}!~l^}gcPi#zQC%PXtA?q4#V#C>&k{{1GISv^IPDshb6 z-UV7sgsCjZOVv!~pvLANaoKSmieX&##m)NE%@Vuy&8If6irxH_sHj4*yi(ze0p-#= zjy&)R)1QEV3;q(TFHpd1Q^*M&7lcv;iBh@BVHCq0LP4S0%ml&>z{8JClHMR*Vi?dC z5bFH=PL7}DK+B<48G1OlOuwx!eo5B$^&uZyUmulC>%GCBTIpNo ziA%g+9riLr{#L|$-?j=dT(e7d{G6fy8;Rx5v%Dkx5G43tW_j~S)PJ4jjgZ9u2Fn{JiT@IpK1>e7B=KM3 z(uZMr1CsdPVtGgK_~-t^;?jp`2 z@+9d5 z_0Ku+$8r6C6nD;R0{xT(F8Tur94GfyACeUc^jhNM??EO(WSlZq7Q56NB8!6sT#!#_ zkTPU54on9l0m7?1!u(cxfX$=H;$*C>;HuS2y?m5K;a1DU1@Je#Cme^M-%2kJKV>PK zMAEp4AP@v&;TXL=f_#+4ZT#Q>_!NRGg8lpg2hO1n*8P3FmwJ$K16w`Af_wr860^!9 zXbHL_F37_-kPKecurUI0>EmQ^eoF&`$k>5|`uO?y`>YsDymTa8ZlYX~3N((4lLO2& zIrtjM;?j{6>6%!qbiRfP-bfbTFAS#>fuHjh;D$U13vR0G9EHg7m`zfeLF)C z9JjcfJ|skteRWrE7d%^-$P*dNOSS=(O4#Zpt510vXn%-H`Tvh4&RF$%aSe4cX*l(OF$xTGn z$&+Q5FPsm(x^gobFon22xn5mx4bnUJOa?YZ1uxC*$3jEfn+8>t+J0CG{V3=8Yj+Dj zgbo(L4@&rn#VG(5Cw~h@CALsNC4axbxGAk8Hs`1B(GG2MiZc+K(~3#{BQ~ezIPUM- zoOW52%tqXR%^7?`5PdQk4H*=|pRhRzA3(%Eu`d5Ao0Iw(#Q&9*`Fl2}x))^TU$Z&C z@oWzJ3y9&v5H=?tu{j^GP~xv_PCl?X?^RT`?iWO2`$IP8KF{WS8~rdgr)$vWT)HZ# z5_?tf7dEFAu{m9c%_+k5%5@<&=NixE0D~f(-U_VFI9YLqlasxRA+J#<+a#n3fAkn; z3C2eGVE8i3e339Flfkfsm%xrz@Gn>&`~_>v;h?!-cZLqb9K;X; zHZw!u7#-ks2E5KfVseG|>HEnr9RfsbL#fxnP-OQ$9Cbdl3D zW;;yP)YMw0p{Z#AwwljC_5_KASKu$+2ab=q%G<&@2wAtB4fdZF!atuAvOn_0J0E;0 zyd`ka-~*qU88pwuY1Ldfs+sWd8oqSi-soA7mqMKXbF6=Le#fRrA13 zAkjdW;P?<;eL#+bvuof3$x>E%tPTkF37X`!YSko<)vH&AP}caY4qUo&1w}`DveslR UxgyYq8sk$>r|*e4po@gbvqr ztmU*g7d3OvO}N%ME0nU$X{!(&deLg}{)-M(=N600Qq9j&{yHbFLNK#gp;k+TwY1$5 z6cJ^uWVKo(s~pa9S)JL`S%^9dRcCf}=1^y0>MUHHMX0k<>MT;7MX9sW>a2`9yF#6n zRcB6h=2Ba3zVtEA52)LCV9cBMMIN}W|vXYuMRL7i1q zXVui%)#@x!on52Ou2pB%)mf4{tD(+ns0Z zXARX^BX!nToi$NsP1RX5b#|RPYp%|&S7$BMSxa@+N}b)H&RVOpHtMXcI=fMw-K5TL zR%h+hS$lQXL7m;A&N`~ITh&=7b=Fy(bx~(s)mb-n)?JI_s~_2B@=v>THlYOH*fq)!7hrHdLJrQ)k1~*$8!(uFf*l*+_LZN}Y{XXJgda zSami|osCy#6V%y6bv8+zO;%@9)Y(*ZHcg#PS7$TS*-Wb(B~Chv-D9xT8@EW*85jQg+z_u~OPh^1JDhp-$guo4er6;|UBJc`G#29IMc zp1_k>hxOQijd%)A;~6}Q=kPpUz>C;~m+&$+V+&rvtJsRy@H*bWHoS?q@HV#N9lVS8 zumkVo1MI|y_y`|k7k1+le2UMo2cP2$e2KmI3SVO%zCkAT<69iSL41er@dJLuPdJ3b zID(`28Nc9H9K&%a$ifNyhLbpj(>R0Q@dy6IS)4;Q&f@|+xEK-o+gZtKwU;L7hFXFl zL52+>5up{Ehg$3rQ5FYx!w`-LltLt;P#R@$1s?iDj*gWQ3-LVj4N>! zsvsT-sETU18i}|D*P=R-Py;nl3(2UBI;e{j)I)uwq5&GB5gMZjnxYx5Lvvh@7HEl9 zxB;!v25oU8ZoDtgfQgud$(Vwvn1<rr=gU7KJPvA+c!+LDMMm&Y5 z@eH2Db9f#v;6-f0OL!TZu?4T-Rcys;cpYzG8{WiQcpKaC4&KFk*n#))0e0d;e1wm& z3%l_NKE-F)gU|5=zQkU9g|D#>-yjqF@huMEAil%*_yIrSCmh0I9Kliij9>68j^Q{I zWZ?vU!%3XNX`I3D_yd39EY2Yt=W!t-w2?}ATlknuEDjajwIATP1Hg%YNHP7A_etOAE{`7hG>MwXo99_ zhU?HA*P{hmq7`mHYqUXI+=!cSGuoj&I^Y&`#I5Lr&gg=!=!Wj-f!oj%z0ezd&=>vC z9|JHDgOG;77=ob~hT#~2bYx&8MqxC@U@XRAJSJcwCSfwBU@E3zI%Z%dX5n_s#vQm5 zcj0c#!CcJ4eB6TtScpZq7mINpmf(IofCsS@%kU7EV+B^?VXVSxJc38@7}nr%ti=;} z66>%Y8?X^i;b}aBXYm}K#|wB7oA44|#%656D|i)K@fu#o8`y?7@fP03cD#dk@g8>I zeSCnO_z)lAW9-6ie1cE$8TR0Fe1R{q7hmCP?87(6#D08>12~B9@I8LOkN63Pa2Q8$ z6hGq^{EA~Z4h2~_f!}Zvr*Il)@H_s%pE!$i$i{hGfCm?0DHR$j8g5p~K?sHf88(C< z6m~cehHyln6e1CY(kO!~P!>+OP!7?Efg9yf0kNovN{B;cT#2ht1@TBgRaC>(NW?X` z7S)l28mNg{NJeecL0zPv9_k|%4bTvc&=^h76wPoQn&W!3Kufg34QP!vXp0+h6K+O3 zv_}Wrf{wTqozNLw&=uX#9X)UxdZHJ4qYwI`ANpee24WD>Fc?EH6vHqaBan^^jKnC6 z#u$vnIE=>xOvEHi#uQA&G)%_~%)~6*j@h^acj7MGjX9W$d6v#j(@Fw2E+t`kG@GjoN4!n;KuoEBRBYccq*o{x{DL%sh zCHCSge2snh2ASB8Z*c$z@g2U$5BL#3;Sdhv2#(@s{DNO`49B4$3n%a!PT~|!;|zYs zANUhzaSquyj|(EN(2|3T+FZ>mp3aq5$Z9r0ULl$Ciabkm<`q&&UQrTLN@B`?E~e;t z1?T^vi&9BeL8NVt>(K%&(F!-9HQJypZp2Nv8ST&>9dHXe;#PD*XLLbVbVGOaz-{P> zUg(WJ=!<^nj{z8nK}f@348c$g!*Gm1Ix;X4qc9p{Fc#x59uqJTlQ0=mFcs4<9WyW! zvrrNN{&Nwakw;4nE^2fCSw)DERoF~fMV_TOvkF^DR#B2wl*E+(Tukw06(J>AMM(rG zi2(n(2++u*r3M$ZIa$S%bAy~8L~#)$h&y5+OG;F=xz+*w(%#unJ+*qj{sRX7lcM#P zQu7~wDI`(%-)XH-iFvT6jJ3C9X)SxKbGfVTtXkez=#eanWQkc|N%kzTFvTvg)HX_W zj8a{rlwy?X8KwG0DV374z#=cO%v*lUQq!~0!j#1A6j_-fZP>l+LmB08jdG-}4P;}1uQQCN?c-MZxLtkR ziIh$6HjVOOyK>m>I&5#e&NINR3~;*!xH(n^x|M-$*FaI!KVFIx9LfZTYl4H~U^i0| zx8GIrJ8fqFaj&byFXHQX`Nlh1K%BmZkmzv#1dA z?0~cpam3ZF^Mjnv=7_7MN>WvS**Woa)m_%OSmtpeu6Fe1s=>a%T0duCHD#G~qw@=x z$wt65#ZF5d-_p9iVv4U=&nW6g)T%Sa5oJ2L11T0MNA<*$BB`FZk|zsZVB zxIWY~JTBL?ge7_nm+(MoJ!JFdch@LjI>}#6wNrx=kwnq;H28myENSpxvR{Mf`M{ml zY89CW&-BL8E+Rjj$9=8EpKJ=Fu?veB7A^JF2pPLjO7@$o8pjSHQa_A}QcR|upXwPR zF|8WHi`81-mbq8bxbb#Y5J*$RS#sEZEVsA8K#D-Xr}9X$9&KizubdY;c+M) zhs)zg__~xi81*I?{55&t7@l;DlPQVYTyOKJT{&uZ9knN1bmmQ}0$j2!;M{8|*_aXV z4p#CMY1+%9j883BsMYiJM3#F)ta_V9InB+K#BE?%=9R3b9hc^pc{fjPdQP>xlqI*- z%r&iGiFOzgDc>dEJXVr#2GovUqve|wwS4m?*%M-llqoqQr7tPeqaS^m1gC|fdo$la zNYL}HAlCIY-~Fec#?f}^ywVg2Mdf>QuyS+opqmMCS{s{E@!HsGcO-F3%Qg~gMCjd> zguTnre_Q+@r*44^!4n*&1c$kT!xAcldxFB0pfFdEi2T+t#Tw?ailTnPQq;3jil#?KNvdJ)P8jC6Esjv!ld{mE}?YEDPr^oCiHj@o=d zHd(z!re1*}9W`}!79wz3E)koFyjp*xW$w$feR*S{J0vYBo_>$Eyn`}V3U?ecPvXnfzMc!Fs;ZvkCQ;+1YZsk|E>sPVD zFK(tJZcUlwFv?m*XpSVJA5!a{q#^FRf~g&B90EqnGi}Vg;Y*hH@W4Fv)l!yxpQwDF z==wfU^e&q!KksLs#I0WS(FJd+`t*8&_nBv_g<7JVh%Cbvw5V;a3zkqX{A5>tvb%l~ z7bLeeRkk&CZ)-}zx{ZW&Z;n`6(m|SEEPX^vSS$Dv)<{z*i^dy`X2LM=Vo(S8$02BzQPr7E8Aau)LY(c zM(_W38*%T_skHA3R-umd09U@7^WnThIqz_tcZiF{*$yS!;mQ_?n9)afSp%XPwMFmi zXeUz=x2Dik|Vt?>qV zx?Ow6E4|}gz2n*Yz2cQ#@vdH?*fU<~8Sm;TioT1G`xBM@iLU*LEX+()G80{yqG&#n z^~;dnca&UE$E?=Ar#jHKPbHvL5tItLSesv0?VbX=XUAmeou(h=h^!@9YJQQ`=rld( zjjbX}EoF+V=4($xR$rsv94+gv^A{}(J(5a4#+LV`?QJx}wP?BTesi!SxV;&qycyK; zO%5aNAfx2(AoG`pXwfoeA=}Y&lFM2wI7Xf$9BN~x66!>F0`0KX8=IVsY@isEnys)u zjT|{wfDu;F?`CjF&?2^&OeYNrj}%@`NdTIa=zPlIlq$ca{m9Pp{Ks9IR5g>K!;G} z8#?t?*W`Arq6eEo6^-s*G4xaq+n2kuH}CnWF4@Cbs#Pg|hM8)6hmfIHA^B~mu**fq zt|^OD%Zpf|pSWW5X`Yx;oF%>4+9;=ds(Bh-{C`I{FL4>7njdX z)!yIQgWKP&^mn`ZyZL}k18X}aDxDHtof4@NjE_A}osfSIZqJ&YoKsrQnnc$%T;^AD zMAwpxHNWV3pMIG?%Nt+GSesS!Mc2BfYgY{H&u_rKHcY>B&7uyV(#M=mZw;*j7u45l4pO3ht&I~% ze2wU&j?(2UDQp$vH*3keu%wHZZ)_y|m%4cKT>oFh5G$=J+~Z6dNa?b}BQ3tZ;Q4`QIz5xa84?m8mA9;3H&iOZKMe{{1Q zt&R}Ayr#5O4Ei@ZKZ|<7r=X!odV-eNKgaWmOKzq?w#KNW!oP&s`Sta7i&whEySl{} zFvxayK|?)_QJ@W2&D7K`pq#>LdT+0vLAEo>n+MryH9pmMfiFYhuUfzE18oOUuLahvD0sU55vNmpOV5lBnY)%*hKgL=C9hc}dp4%Q~V4%S+=P5CMvto}#n zU*al%K0{GS67>;SMq9;y;wUAjcA-Y||Mt*f7s@A87y zTgx)-5;7(8#f97*zln^x@eM`QFL~t+U#p2e+B5fhTOYDRoYOlEE?dMiKlKWpRKMIT zx^hk7uQz2qBjGEddVJ>Hni{k`ycm<(kLyWi$Y+jeHFwQjxO$*uZVbu# z=_{f}pJ$#n|C0QMAep*#{e1yL@7BM(y$Iry*Y~s-t!L`Y7GIFn>pZGIy;|UX7Gt-Y zA4_7XcjTHse9>8B)P4C3hkf#s8npbklas!cBb1ipr1^!?$F!W(@Z9r3TQOs=Q6p04&p2LiN!StZ=E0PH~M(d#C2tBSv6pxgXI^tNAz< zj1MHAP~`V2QwA&RN3}U%r{x#?>9@@}<><+!|K3;4U#8E33Q)8^^a_=5D!wnZC`ULg z$xQPLr^coKY;QmnnduB)I88QRry?`;C6oCislK>)sZpttdGNr4X9^6Gj9{%9#MG;% zKGcfFx4-7Kxua73F$yB1KGf=ayLt40-cc#ifTHzN(z9!PZJRw~oL{EUB8G<>%T1x^ zulwo~*VPkHc%p?h~hJrUpR)|1dR zIihAs5}IGs6iKM~mT6g$g!bUO`cYO>&@^9uB4`#dCXR1s#6;p&LBmhAqk-Y4z9Y@>fg*8;Q7 zll)aXiW6?+`?~7y*61NI$zLTPhTHB-WJ=;zui_|(T28Cb6Ly3a-1lkxwP;=D4sl*SU~i(eqx%=9l?u86`jC z@CjA=dIA3MlxLExOp-TD+9$JRtp!b`s`FFY+A<$e9!m7s;^ZTqR1DSq`f;B$P$g5R26wVhKVpB*?HK1fj6QfiQ$40;Ld%D3nGST!FH1 z!i92(Mhx62j|zxIMN~o@D&tCAg(`?g0;-}Ku0|rR!L_K4B-B7n)Iu_9qYmmK1@%xL zsc3+PXoSXSf~IJO>(Ct6qXk-`6>dOlv_V_kh?{UT+Mzu<;1+bmt>}c#=z^~3hVJNr z+t3re&>MZw7yZy5127PSkcPn+f}t3O;TVB*WMCvlVKl~IEXH9xCSW2aVKSy*DyCsN zW?&{};dac%9k>&B;cm>qT+G9K+=B&Jh()*;i*X;8;C?)S2eA~(@DP?`1y#!ahun|w;X*`2x@f@DV3wRNm@Dg6eW^BPLcokdm8eYd6*oHUp z7T(5oyn}b~9(Le;e1M(!5Fg=V?80t*f=}@o_TY1TfiJNaU*T))!#Bvpete4qIEe4? zJ$}HC_z8z_7)NjvKl4BJ{$hDA&GIXEj^Q{IWZ?vU!%3XNX`I3D_yd39EY2Yt=WziZ zT(pOHEdSV~SVGir^-w_JO6I>S7SXEYX#F=Ftt}7#6-TSZ!cl8Q5P~5=h7BPIg&hur zAsi7Xg-Ar9G|J!#l!X&6ltVOP;6`~=KrAYv65>!9SK=yEK|B&r71eMx5^)W#MRg>h z25O=fl2IFVP!}nvhx$lG12jY&!Mk`5JMca}z)pOK zkMJ>eVK+X(r}zwe@HxJ~m)MK1@HO_~8)RZXzQq9?#CP}}Kj26FghM!tBRGnm|EH6y zQqDD@=HI#0+k&&ywqS_b77VhMBv=2YroiP*uBvz}-v79jP6CmSw|1Vc!66f_bst1~ z|68@7i|X8B;Xi}VwM29(RH@+YR0y|PVyyOXOORaJT9MubNe;Kjvdv;c2tr|p17Qe9 z1WF+iQ7DZvxB_M2gbU>mjTpF59u*Lail~G*RK}IK3RMt~1XM*eT#ZCrgKJS8{AjVI z25O=fl2IFVP!}nvhx$lG12jYa%+J};{iN~ zrC5fCupBF}5)We)R^t&oipQ`9k7F&Kz>`>q_1J)ocnVMB89a;U@H}3?i`ayh@G>@I z3tqvi*oxQiI^MuGyotB)Hn!s(yo>j+1MlMl?8Jxo2p?k?cHK_>R&TO7bae24Gx1AfF$IE2GEf}{8uzu;FK!*M9c!U_C_lQ@ObID_Bu2mZua zoI^Iw;{rUmXti0aR+;~+Ri=&h9{&^kzmsIGRk>$7}7v)~X-+d`4TJBW%hxyx-d6^WHB;Q&1%}POy{)$&)er10}Sqd5! zU$*e?_OEy;`ZlGYd*uUxza%MWrQDhr7PLwZ5ByR3lNEO^1-&M}RP?8nf@&zQk|bLz`4*V}NChB;d$R~* z{hWW!?_vL6mP=A2Zw_9FmoG_e}ZDOVS9rOwnG6Bu$lj6z!qrk4};n zdK&|U*nQ@Q7gUy{wetMpzruo^;;;OhBz;lX1FN8yBS}Z)Nrm4VlJvVbJ-*cEBU>9v z7MbnmAXmzHKlk7B-k0Pu*7U+Y$lv;sT*Z4}<#;`LpU>Yz|EWWHBn~#w zaIi*jhieWtt?eMkOSTX$KZMdXX@>)02uB1;Aret2jWW0bW#NPi6(G1t2 zIj%Rg&RjkBMbpCrMMbcu%fB4;(a*D5<#A%;=%B(n1xu|0y z2!$OEgd+l_5Q!+1Mj2dzvT&jtq7ef(%A*2eQ4y6OzM*g>u0j>W zBLP)W4Ob%(;@b(;k%StkiCRcTZPYdOl zv_V_w-$-ajxjj1I7Ieg|=!DMDzm3qHau3{wp6G?%=mYaN5e9O95YjLhLogJ>VEz`u zNbZlqXpF&FjKg@Czkx7?`%^Iu(=h`xF$=e2HtxV(xEpgY7xOS5_h11QVi6YOJ}klg zcmNM#DVE_OEXPVbj8#~TNAM^f!x}t}wRjTiupS$*5l`W1JcDQP9A3bS*o2qxGB#ri zUcsx_ir4W5w&6{@1%KnV-)H#;*ohDE5&VtW-ox_G@ddubUicfY{VmH6;2^$3eq*(d z^0%M8k*h9AWv88mQYslV8E4hqPWutVh zQL1N@nj58?jZ!zGG{7ij7^TTZ=}x0`uTffIl%6n3&l{!JjM5IH^qEoGZ(Cq`%(g@;+`>y4cLgM@HC#mvv>~A;{|NOOL!TZu?4T-Rcys; zcpcmDCf>r^*p7GbF5bfqypNsu5Fg=V?80t*f=}@o_TUSAiM{v=Ut=G>K_>R&TO7o9 z_#QvtNBo3CIE*7W8c2ve$%?0N2EXGE{E4$Thish31$bbwhwSRc_ccpt;WfetOVLK@ zDx;KSZzWEZC)J>wYHzANs)eyed!y9DC=KFK9GKdpMj4NqYOHa$QM%74J#3WL8KoDE z(i=wU1Eci0Q959hezv#b%h_Ai7kt`y>WkhLE^oB8EE~Ebqo`3lRQq~IUFSKG`|8(R zQhMh|Y~OdcULfM`Ml##7=D2&(Zjo{X+#o*NV92l`1fj6QfiQ$40;Ldz(kO!~P!>+O zP!7?EL3va_EGnWB;!qh^;wn@@0;-}Ku0|rR!L_K4Byc2MGT^4NP6ISTV>CfiG{bdh zj_c6^EpY={qYc{PM%;v((GKm=0UdEGI-xVVpewqeJ9^+Y^g?g+L0|Mke+EXH9xCSW2aVKSy*DyCruW?~j@$86kzJ8>88#vIJUeB6TtScpZq7mINp zmf!(Ah^1JDhp-$guo4er6&}H(cnoXsIM(6`Jc)H!kBxW=PvaRpi|6n>UcigkgqN`y zTkr~A#a6tA*YO6n;Vrz4?RW?8;yvuZ`}hDm@ew}8F6_o9_!OUE4?f2i*o&|5HTK~f zWMV(Q#Q_||_xJ%n;wK!!VI09x{HzAu&jbEphd+4GpE!$i$i{hGfCm?0v4@DHRg(BwwsJ#bl~zXS7NgYDC=E7BV~o;tqcqnj-EWju8>J0K=_RA| zrcwHkfX)Y@1@teCsQR6;#;-={cW+4xZ;QJ+k46yQM2_kG@taoF!iJ{z8$76bYSOS# zgQK-OHN_p`+y~dx9L7w~FGV|khTYOAD2#>b!)l2&V?#F&L8h-Jh2HITgb+IuU*+e3mY#J8@V?&PJLGKBJ@XZj+fb-yqtHp!^%70A>w)O zZM>zy`&RCFpR37YOMK1Lb`$p7Sdzw{m&MGHmo{!SBhxmciaaAdc*4%K zbEx|$zI@Cm?rh}=dAN9M5A)WpVFzhD>KuQWpIT1rZtbzgUb=z*_Cao5M;321)5h&q z?>=gIO5H~**bsZ#`>|EE&dvy)!KM~8o+I|tB3_1iknrT}KVx5UcO$E+bzv3;l8sq zS)8u1QI97VER@v!a|ge1b{pXB*I|bC>uqIri@&W{3|}oO8=!ZWJex zJXbv?^cQWMPV5l%&~08tZuteLmiL5;%X>nJPg!Q8#u06tRoc<$jX0c5k8(ff*~9{| z2Ye?{%NDUwEnA`$VbXLG@qQcVXV4aLglT7x7HI-JgSM*CrZ8s^uTVss;+#PfMh3UL z3@6d=ybN&ya!}<8?lUriI_cmm8t7R)Dm@t*kwFo#ujVK zTT;7<&E-5s_inLrhCbBQxujI!_2|dfv=(P*-tL-Kc5)GWj}{vpzLPY$j&_oI&(B<^ zX6>5RC`*{LFDIO(Rti4}Y*f6|Q8$X?%v9ltb1voxx2CgBul6h{*4J{^w>eI70;f2} z^|<*Y`!}GIsb!a#$d;Ya3Af@0d><1j>cC|&)%~4R`fE}vDxTGo9_&4zVxGTxb4-+} zhBAju{e)`5N7XK7o%pEcQ#tt0D7H@z7VpMe*>J{E=2UBWCP&4U<1D4#;RVlcJXnOO z-1XODi&~8sd*?;geK>V$-c!$8Ltnb?$_P$Y<5gi!z6t92s-1i@_SbLu`{s8hF&_taG{WRVGz1NTtR+GZO~iEv#l5%wj$-h9{Di2tQfuT7~z zsaI=@8&LrhS^DZJ5h-#r|23?d3bP~3^=}hz&euzRgwJC&@=1C*;UWt<;q+%0yO{)& zk7~1Z>1J!pqr6Ct2a!>$nPKkpPOn(J>EZi)hT?qiRb=VGL(^kAOyQ&TrG~8Jt;}NH z7e@?dRIbXbIbk{`XP91EMUEMD(tBzZP^-lp5gW{t7wWNoXQsDi9mZz-o?NLR>JetV z2~K#*Kd`=bW*r(xZTp}^wm2wb#uh94UQG3 zb-h;6Qd#jr=aEi(PgOqtW~L8d_Ke_ocT6+>7GFVc3$*+_N5ZI|ex_zfb6(~=zbB!V zCC}DB81d;AbRq`crr8!R(?Td}n?{XD6(0{Z+A}RX=WWk)|J{=joR$&XvO~;Jaj@w} zvf8NWm>#UA$+HV_CV!??!g-QpqY^e}%JsBLICoZ@^Az^w&^b>~Z(8j9G{CgDD^>Xs z>hl8XFFt2+lIpc_&OA|XNSKZbKK1}BFFkcEPW@Hv%+Y`4C8{B+v31-w;qu?|`; z_TDX=l3H&80Xm=j%h>TeSk0T~#yWS%OK(wzWN3alSoLyb*IX%!$|6e4jt_ zo|8Gg9!@=_M{6S?ud4Rf$P3gysC;WIHc_aajFUz@)+#J2E9xe5D(H!q<6~n^`=6#zN zrmsxT9Fh6;=3Sf9H}8|%kUFgEPR0l$C6EF#G|$g;^= z&uTB5mA&o#Z8IYrQClc|ZoPtyWF61m&0~0uhOup>?A@{P(zZ=JWU^(qct{jaTroSs zUY2LiZkJtpWayTxhH{yRp5eT%c;|K2LC#p#tRIn3p<#9q`)KzIe7WL9{*SzH%m2W% zB-}gHtBvz~j)UWb`QT8h%=I1|-UDHW_drNj50nw!17)B0K#|LFXjsd|ZQeJ6gC{-n zYq^|dp|xIk+R9xV9(>;Q<>=7#&Aa4Q(#mveE2%t(hUEp0nGu`h_EK7=9A{4oU7j9K z2l6I)Fkho-)jDOf_W$!PH}XlFa#lDlijAjJjmYCdQv<+v{1AOINUP5Lr1a z#HXtrLe^U?-UBF396(veXJ&7+I=n|wX}h#7%j%B3hoh(lhlDsV;`x=TG7-HjpNk`n zgU7x~u4ic&+s1qBl-;(8R}*g)D|1+>M~{`&xU9;$o?5lodw`AtIhnj?a|?gL8i>e-cNah=ay%i5G{>)xmO_ zw0oO8Nv?0rjz0dscTlJ?<8&M6BJs1DMul?B95c>U8R0!BydSr>3R$^NM2-|ba;g}6 zx#$5@G@JLOQKNjkS)BV)Z4Q?TX)DubvdRwP!sc|jK8KE6nU7u^2Ak5;I1EPMYxPjE zrd3$UCgV4!*S3z5JIvy_c8$pN9uVRK76D>6jPSuHEb+O=NXN5%ale8Z_qS!q)DhkT z$lA;}D6+&6lTAGlF^KOS+WD^C$WVe1q2eA(Lu-Aj-O5qo4M`#Z@p1_!1xKK8aS+)# zQsmW^-LYSC@Wdu?7+GtS{okuwHhP1{;#$tP$Og}c>50b&k}2`{Ao`*HBJtpuSScby zdU@|*!HF(H$R_VGljc2UYGJQQLaJ?kGY<8?UgMftkf{!oA;$` zl3SG}bJ>-$-I^vMzK9N+vKTHWvIA-k?U2ypyIA6_TpE%FkbQ8}cyk16J$t-0My5gt zt(d(nW{}h_TOP#MOVdIdW*>;Tj%(H83?D=W!upnm&iLq3|KC2tSTS~+Z^k?w$BC+s zc8LzXNY|G5$G3v`$szGid{-#%QvNU}u8RI$p-_u#wOAuUEkOu|1Q|AjAQW~u5QcC> zpcEnzh0-X4D^M0rxKIw!h=CjBQ30{2h)RecN+8+xJ_dZQ2eq96KW00v?Z(l8i9FciZu93zm942;AmjK&y@#W;+| z1Wd#vOvV&U#WYOE49vtV+>Y6}19##s+>JSyi+Pxjd$0ftu?Y8KG48_>+>ZzFAeLeo z9>Q|0z)C!fRalKj@F*U`8a$4*cmhvi9oAz5HsUEfjc4#Ip2PEa0WV?`Uc$@Rj4gNt zuVO1+!|Qkh+wdmd!rR!6cknLW!w$TU53my-;v;;FUD%CJ@F_mS9(;~3@Fn)*D}0T8 z_y(ESk8g1R2k{-g#}D`sKj9D#;|Px8XZ(U+aSX?yAPXn(8&2XBPU8%I#~=6;XK@bM zIFAeP;9^9mWU(K*$S<36F9^YqAj5_bgu)I7!Vr!KltLt;P#R@$1s?i zDj*gWQ3-LVj4N>!svsT-sETU18i}|D*P=R-Py;nl3(2UBI;e{j)I)uwq5&GB5gMZj znxYx5Lvvh@7HEl9xB;!v25oU8ZoDtgfQgud$(Vwvn1<rr=gU7KJ zPvA+c!+LDMMm&Y5@eH2Db9f#v;6-f0OL!TZu?4T-Rcys;cpYzG8{WiQcpKaC4&KFk z*n#))0e0d;e1wm&3%l_NKE-F)gU|5=zQkU9g|D#>-yjqF@huMEAil%*_yIrSCmh0I z9Kliij9>68j^Q{IWZ?vU!%3XNX`I3D_yd39EY2Yt=W!t-v=NuXy#MqRLCq?j&6QP% z0b~3lt8l0#Z&s0KDStC%6^@duq9mr2#FYPBOz~wEVI^5bfe}Dz9oD8y5tA?(Q!o|NFdZ{66SGi~R{V$4iinc5qQJDGBmxj2=Ak63 z_#ewEwAA3DHus-ZFtlF$BdaK-mb_U-o~8WFlvR`}$tp@>N=a7nAI>TwOR|cR2v8CM z{&Nwakw;4nE^2eKimmg5oX4Wf1D#7%n`<4=FYTQT)l;kY>px)NKPg&&DK-D`mqHS? z>Mm;qzQ|}PV-5DCNtQ-)DXV)b+57VNDM@~lF=m0Kwr7EbDRzOSj!~*>lv0dRJ)>0L zD5X+T7Fgs3mU$2MfRF!S~48JEUk|~8-ZMDS5 z!u7}U&b94PD(t>eH(aROHM|OaPkN?B8B-eN2@Yj~!!^N?P$Aqi-l2?lxW=aE)~& z@W#hDlraw181W7jx1zXRiksqbw{qO=IxdRm9m;u!>%3UuS2t4SjSha>no-hMg4GEHN?;16iATNk0919QqdPhi5K3ACBG%^4?C61S%4JB)I=gDHvIgO!Rl?d5?+IPq3F zYRxc=@<6vT(Cr%NPNZyl-u{$}`gH08xAJh)bE)O+Ea?%i^oV!$h)>uNYJSEX?{s$! zPXJQ`)?tGIOv$V>JTBL?lqGs?masnDRJj)zZ+CoBqxAj@dZ#rmj(HRrX_@{+%OJ}0&w}lTwYj}v3Z6tOpYrJ4J=A{trezGe+*jgZdH&1-dsho4V z&N&lqDrJrVy@@&_9m+_DYosG}?PX8=n()kh-AZ4#tFJqea;`k{h+R2icO4PoU5|?7 zVT0VtAh&Cfo1&fuQ=I5vO5!%3WKUgwGQYG-tIiU55jLA*v+cC;p&PA&0>e+PWKd*$ zT3adMT#hPYn7;tLfx9gZ_d0(ozohvdoAQWrw&-{u+8#p~K*R+}?hwREByX%mh zAQ@OVF7-(b*3DBLEXl75@Z0xL)cs7ug*dvNMD%cuI9ig3<`+jFxYJrqbd3hj^hVMy zlBGqEDUR098Apxeu~4y*&Qzn(?rWZ#!mm-`nsZX4^=z5@<-@*&F~CkpLcX`N^{y<= z5&!My-1!E1wiX)`%ZBTnwgkP)(P`^9D0-Lm3Xiwv-AhT_>eU^UpsBjkD+ty!ZLf%B?c@@Tf@}v<61U#gfoKT* zXjgu;yMDA2Vc#NCU7RDrmSn2=Mc7BROtqpfQw{M&*ktn+C^FST^lzF1sKsyoT3h6= z%0hkx&@%VIGe!s{&P7|98XIkHk_%s(+g}Hyzw0=9o14uYmz3kuyyFSq+qt7%?J7jz z)DJZttX~0e>$xzS(a#tEnYHiuG+J%NKMzb>=tB8|L%HB^U2r4}j5J^A@V@S3ZvUDq zGh=w>F;1o=ZcS};a~Es^T^4zAtpJy7^_P+nh(sUkFK(tJZcQ;dN6(};mKGrZ7Sso; zr}lpOU<1oN)fBZOD{BXA!dIpAU{8G1`zk5ssnJA-z-={iO)FTUU6f6rKHOdO-q?&R zk(h2IFoS`Ba36mmto`$g=e3UYSr?z4OZF)4{A#c3$%glOEKq2BLx?e+OrtUYJ5;Vm?r`l!e3 zmv9+a&%hGm%6QV9O1jgP?j)Lza4I95t`VX*+^GzAx`vCQo)uHn`w=J(aVkTct|87> zC+E!$SCchr9n8Jz=*?|vIDgG;U;U?r%b10EH(0182M$(xb#~s`NVNQajaPn+cl{cl zP$$A;8jE4-WY7;I-l3_Q)9VD5UX~iKXH8GeX*GBCF4%-M;Q@wYwGTF*S44+1QFRTe z=)P;FCUJl=CLQ zL6Gzbv}HVbL1tfkas2BDr~BX~9FO3frw){Dj6$0-`5K369d{ zX1wUaJZx7E+g*pnFeAN%ODdR8(o)#v71@qfe~3<7Q({d`arT+QZZ44Zx%TYo?^gP| zUH#o>H|Ku0rZKZ=ZY9m_N)scg2D_P(xGhW{ZbyHOdC_khTG}^u%JhzN^xFpBp3l?6 zyTW|!%G4wg!)Nt|Lt_6NJ6jB&HLY3va>bHHeuM1`|COb@ z{kw?+Ojk(r#H`gU(VnH(+xl=DPIq6MR<9!h2`quHGmPieuN;e>TT>9|O<7MyxZI~F zBRQM1dEQ9iPtBSE_u3XF*qI~aZhwK#Z!f0!{*qDk@Xr6<4%-24Wq{i?z)gqkr!@kZ zl36XVeP5B)cyBo;(Pyh)g7w#BJLZ!bB%R`#{nsT)XFy4(e$77G*N@uN6g2DR44OrY ziS&yu)V?7~%4Zb)hTVK3p&mudDWlQ+W}yA`p_eBTn!eqacbK-Xe^-ny^*5%p=pDR; zN*K7tYV0R-=xZ`9Dj_(`7`0Bw)4%OPQBPbb7B5j@442-wDfWXY^!|0DsYF*UpK+dV z^yKKu<(23yQr@KhuWy>Va({Izzq(z&iZNEkAb>onps6D_XEH5nAnh6^dA+Z``y2Xj zbH+qMq^Jb+dQOb2{VV!yzcB`!-kl?~mR#5O+p$VMY7992!y8;hKH9_-TGgK}(7uqr zFu!4@euHFTM)~^^Qd7MAGSW}DB(`n~Qnm%P+{PJE@F*#N9hP}h$AoMLA+zwKr2LJ& z*GEp}%^p^BBI@aL?zX62ZRCAaah9~>8TIe>o5SBC0?^+(m{c@))7(r++~$d00WNuw z2j-~}ma-&Iv|Ygx;|(6D zvi=n;M#Sy*KU(U^-69zkUv+vVN5m|-sGncN)W7QF@+G4)eGxO+eEo?QylJd`(Z2PR zW7OynBO(&Bv{9o(L~V$G-pK7<({yYV{Occn;$f%!Mvdyxji=U!X1n#Lq&4kttdy2# z@K^n0GUvv=4%y-aI`dfNm3d;CRh zc<;!s;jIt7&)dhE&>+;jpDtHfbh7H*7=`X+&8IQGo#k3@jrbm}{+(O`qIZ8K`Frzp zw{p-M-&7?M>yxN_v~P?gq!woURGx||kp5OesTfqcBS$DLNj~!nrAB9Hwl|oHFcg5FeVl}kkmJ9O0R8~_tl3-F?b-=v(~DtwZ^aAx5U@1 zCYbqaRr~vAK)7xEGccFW=OkO^tRc zqn)nN&ctOoF1Lwak1~y#EowDT#QxU@O8dSbQTV6MHDK1Xzr=plYgT@oLH}SWh|gB< zgw6RY2Ava?&WWzhiJVi$hi|GDntUpf(^p7N=jIq2Rg#?M7f~N(#HslHXjze*_8^`{ znSS-jJl>S(R!h$t2@#idkf~+f%aho;)(6gKdwE*d`Vqp*&2T6g4p)XF)oluIMK7AR zY!r=-hy<=JU6u=2rhr*cx2^cS3{#*M35?cb%efQaU^Cy@WNN#bD+a&iz9vsJ%O zWnS2ht?A=f|9|z>5-Rm7mh|&~x3AXJt&nnCO+xOXR?Zb=7>(+nY;)Qwq*$s2U$k6| z=D&ZvNOYNVl_*(eIjq)@P!?H3ELMAnB?!TgAj5_bgu)I7!Vr!KltLt;P#R@$1s?iDj*gWQ3-LVj4N>!svsT-sETU18i}|D*P=R-Py;nl3(2UBI;e{j)I)uw zq5&GB5gMZjnxYx5Lvvh@7HEl9xB;!v25oU8ZoDtgfQgud$(Vwvn1<rr=gU7KJPvA+c!+LDMMm&Y5@eH2Db9f#v;6-f0OL!TZu?4T-Rcys;cpYzG8{WiQ zcpKaC4&KFk*n#))0e0d;e1wm&3%l_NKE-F)gU|5=zQkU9g|D#>-yjqF@huMEAil%* z_yIrSCmh0I9KliiY!8usvAmaN`IS4za2yJ z+Cw~+e{51LA!@jKD4=j9he+ONRdTfc8;;hNhyRMB)nehOwIT??kRZc`5QM@G2f`4J z2$Vu3qEH%Ta0SZ32^Y#C8ZmIAJSrd-6;TOssEjLd6{;W}38;!{xEhJL2G^oGl28LR zQ47hajXJ1{6x2g~q@n>Dq7fRS37VoAu0wNNj}~Z&R=5GJ(FSdCBW}XYXovRbfLqWJ zx1tj|qYJvC8@i(hZbMJ>LT~gzU-UzN48TAPLK+5R2!>)9hGPWMk%5sIh0z#;u^5N( zn1G3xgvpqKshEc8n1Px9KcZDha#eD)ax?~Tw3Z}S?5DS}9q-^>yoVilA0J>RKEy}( z7`w0=pWst`hCTQkU*JpZ#aH+m`|u4iu^->!01o0ie2*XSBYwgm9L5nG#n1oK$yF)m zno#pM6ZN*>EVV5dqP7KttR>0Szo{v3d6TOu9?M11B|s;E$j4hd&(+|N3D>$0qHX-i z;<%{JL&bmEnrjJvo>S#oZ>K`IRjeFpl_fHmjTpF59u*Lail~G*RK}IK3RMt~1XM*eT#ZCr zgKJS8{0ADA8mNg{NJeecL0zPv9_k|%4bTvc&=^h76wPoQn&W!3Kufg34QP!vXp0+h z6K+O3v_}Wrf{wTqozNLw&=uX#9X)UxdZHJ4qYwI`ANpee24WD>Fc?EH6vHqaBan^^ zjKnC6#u$vnIE=>xOvEHi#uQA&G)%_~%)~6*j@h^acj7MGjX9W$d6v#j(@Fw2E+t`kG@GjoN4!n;KuoEBRBYccq*o{x{DL%s< ze2y>hCHCSge2snh2ASB8Z*c$z@g2U$5BL#3;Sdhv2#(@s{DNO`49B4$3n%a!PT~|! z;|zYsANUhzaSquyj|=eNqSf}qPRU|_kH%U-29HJezr4$+5k-`(JEMa(t2BW+X?ZH+lu|-XzCiZ|DeM)tv9)wA`uST~Wtz zL8SQ(kM~wBCB9nE?QoI%Ash0f|%x=sA+`=r~#6i zrv5@RO~oU-JW|W-GP^Vi%!fzQWh|2H)a4{2Tl6Jr3X?e!!17 zgr9I2KjR2~!BPB*V>pfz_zjg17Y(I~vcs;}i9L0q_txxwGE~)R^}1&J&w6z${@Tk) zvrkSLU;8TOVmIe156!;Ta-*Wc5*6eTBwL56`kL29o zuh4V+)VDOnFGyi9=bYR}TDTP#%8r$|2IRG@L63-)g^Z z2r#c4{H-g8swLveA;kD~gRlH`gTJ_P@Hej<{Kb`ne=S!IN&}`FcKC4>ahc{Hz*R&b zFB>8VjnEhl1fvN;;6zh2Lny+~9Ooe%EpR?AKufp~fmUdZ3vm(Jpe@>=Jvtx~QHVxI zbV6suAQo|mM;CNO0=l6)E=D4fkc=Mai4>%w7t+uhm!J>&;!^ZOe+iBY%$SK=yUU^Fr@23g2P4#r{}#^Y+_A`khPfC5~DiI{}RxE50|71v=J zreg;Fh3ip>nV5weFdH{w4(8$}%)@*vKoJ&V5f)ybh6p!I?Y{Vu!fnq#~r%-}YY{t{rf@kn7 zozl-{4z(hks)~zQ+L^#1Hrphwu{)<7XVfFF1-{aSX?C0>7aW zTsbHKA_4>u(ta2Ha?kCwfJW*h&jo+=bV$I(BAR-fx)#twP8ZBYT0k$)m!50>RnuC) zWS75DpIX3F`MVYCWM-V{WVC=edQ9z&K?_)X=2e!RX_cBvPYbwDPQtCjUGF|zE#P52 zNOb+$53Qz5fA2Ca;8lJ7x!auEndza5*_z> zd(Izxm7yhXeAk#k+e-131uI>(lpN-T$H}iPFpn@ds`ORk)RjkBcSo+`nIiQ&_kHSi6}&)Bg9<_F^EMR;?V_Nk$`UK4snM<5|YsaJ&}S`^gcO{G_ory8XLN;Xn1O%cdK7{?5v=ca-$;56=He#I!+b125f)+*mf&VA#VxoM zx8Zi&fn`{ZmADhDuo`P{7w*Pd+=F$v5BK8%Jc#vp2pjM)9>Jq{92>C-PoNl2;whA% z6q~UH&)``+hcY~m7w{ss;w8L-ZP<=iu>&=IfBQ|AzlFE42k)S!?{1f~yaFF#A3jD+ z-`oC*sZ!!9j6%OGi9LCQ$f?seHzv389;5Ss_KTvG-imvpD8OHCJ zZE;(TRAXDJnL5u(U1X&?Td71V)yGN=wNe>YD%VO)wNkUK)M6{O!b;t1r5>|VPg|*% zt<)P<>O(8_rIq@@N*$M}qEb8O`lmlnztV^BTG8KeqFQ)eR6V>NXchU6h{#584f3+W ztIgI{U%$ai5o56tWWA_IZ)nirgL?3V4I;?e;Rk;NAP@}^ghpr#2bv%RPBcX`gdz;h zaUQ~PJ}y8@xDbI>XpIYT5k!z}hxX`zNJJqT9nlG$A%bii;?V_Nk$`UKj*F295oCLy zCsL4#UPwc4T!KCjLAD?IV*mzX5C&riF2hiyV>m|Oa*V_%T!AZb6*4dyV~~YxfC5~DiI{}RxE50|71v=pX5e4A9)*~RS-1hSaUSf249l?sE3pczu?BbHZmh*USciLYA0EJiSdWLW0T1I5Jc`HgI5y!4 z6yr%eg%XrvGoHp4Jd5X0hUf7DUc^?sgqQIOw&PXoz-xFNJFyG9@dn<++t`D5@GjoN z`}hER@gd6b5%%F@e1cE$89v7s_!3{?8+>a7*qt>7*dJKoNBo4t_!&p=3y$Jf9K&&( zKqdYI#a7P@rVXrc6l|qhSgE#FD%MIRTd7N})G#ZRX{GY5)HEwK$4cF7rS7y+_gkrr zR_YlmwarSsWu+>t)Yn$(kd^vPrvCKMYJ~S>8DMYi>{!&u<1><&E2>RjcQj2~LIl-| zHf&fI=$^v{5oO=AQD?S7dwn9T$1K*Hy40Y<2le0!8|tG0?C?hb0?`mbXoSXaAQ(;H zL{l_FD8kSj=OG*|a6VeXg$T4lYg~wn&<1VM4jmAQC`6+pI-xUS5Q{i;L02T88@l6S zBq9mP=z$cZq8HN88<(ID`r=ab!vGA#APmM3T!x`Y$1n`X%IT!(3xj(_2L6k;Z3;RejcjhKVEn1}gTfFdl! zA}q!d+>E8T6}RDb+<|3SjulvmJFyCDa2M{zTHJ$mxEJ@~emsEncnBNtFdo69cnptY zBQ~KJPvR+*pcI?&G`8RwJc}|sj~DPFw&Eqcj90J?+pz<$;dSi9F6_n|coT2oZM=hb z@gCmC2iS`bQH~0HgpctFKE-GF9ADr|e1)&^Opfz_zjg% zZ1r{x<08}F46}Y#Pzte97g(wGwp7to9x!IQ@wSW2t$JE(^tVzYWQvv<#}2Y={-SM~ z8x~j_&ahU$$x7W~rB++12d&f-R_Zw`^{SQHW2N?4sc&tm{3(nr#xWmu@7mY6b?w%4 z$00xBoS0O~zkHM0y1}@-jGU}?@$JVK*l#snxzC-Kx-JjIBoSn(7mp5oO2#qPL{|9r4ZprS7b3)hRn0`-MhgvD5b zo5KRTELD_fp0wzwn6*YyzCQg^M}2Ah$}RzlewVYkzQ!4#uf{6eiIrI44Cu04NiJ3O z1>MrSZ&%rEcn!OqU8=I*^4ME_uCf_?I-+KxxuZk}cIjtloBM0^%j%abWYUOwsq?jbdik)xPViXJET~Rjgl)U zH0Q?3{XS*;ee}EIuW+^J*l1>RXfjEU2ckpvpLqyW{(b zgImM-wWXlU!s<7{+u6L3x5=P!l@2Yjxp0#$>Pup}7UA3|2>}0%+?=6VF zYPb4oR436{!vdAg^g<;@JjIHqIPnx86G*QU{Zok&PtoG3BTtlAjlGYKr!?rSoVn}q z-HrL(=83{}_j*y6wRW6R^)ZU%_w*X-uf0*Tb?P{CMOD)B=MphQm5Ls>+}THTA$`8l zBHRaUbtcj3WW!Bvsi(X{$CBMC{M7Cw>R2aJt36wN%5017zO8x^zBWdR6>-5wpZdA&QJG3QN_Lf$`}gyPDc) zx>w`q&Y~aDO>a+kR95SzwU4N;86&FaCpCpOCDE&@hepSmJ+!Jzc8pX5l?IMP`k4|H z!ymwm76WSa;g+KxCj;hk*3suXxAW}L-Q1&{5~HrEKFrqGvBmf+->_oq1C(%`x>L$f zXF$qj7=pnVgn`ZgF$OMowyUXIxev}5sMvL7p115#&W#tAI0HRKD9^Cz?o!>UtbWc1 zMP(H+IMh6&*m}J;`EG3~h8((C%Fi_pS?&>sLoMo-BSMrp>RBTfd3cODMhu`EM6<=} z_!%CZ&&*1W(Ep9GYl=%}m==XzG~VuV>=Gk><-NZAbidcvJ#bZSR=d;T zMp$E3EZu=Kpvhaq)mj#%yu&=9mlydR>8y)skSKchq{=;FrSy0HQgffPvJqVavVZE0n9#9h?2@J_X+0W`G!1>#f_c zfswS@@Op(!N${8k%YMr=*qG=xHzv9#4WvpZF{D~!FEg20%#Oor9$l?Y=-EZ7T4s9} z>vvNdci}YCUAkI8ZC%ZyK_@e{q>@V1WhJU}E#o~cLCmF%IpN7Y&ONBeiQ#r?PxObX zK`hUET%L2lbSkxKQpiejAaiaPZG|3n(m<9>h;~*F(aT!u&Voa-Q`It}IlmSeo-!kJ z$N3%QPBE&ECz~2awfK5$ZnRJ1Bed^0GjI?)X%oDTn3rkg_Td~Dfml*Xw z5eI&roK6=OdW#MwrV&#jbh=49_lRP~0db%Um`KY`%Ak_yQ310Hy5e03G5o&OCQbC_Q$NT{l^^7d z=((YO)kqvAuZf+3=9Ji)4WHDBt0vGCG+$0;O6=HJw<&RatQHj&vt5kXrysf~Rx-Al zYV}E?`x?TdVNmaN8@(uJW8Qeu&}DYx-{=mhK9N^E~sSe1ldvd3$lphd?9%Tj%27_0+6N}4zT+Q4c6mY6c!v>hs zuveJNiJ})Fi3oj~sh3soY)pD+smjnhCyY;?7<#=(%TJ#|YQ3c0>fgyFI;;4?i2MkO z0k^03dXFJjeon=Z8ztt$a#DPTkDum=xpCDs9fMt>Qo|h2bHL>U%$M}Ft7?2leEzil z%st-hQ1Pv59knQ}v-%u^UOc|qL@?_}SYXyKIEr7x0t>5uoWa=T3ge3PG-FWFnND`r(* zd2V@j$*z*T61|DNJa^Sz)t}3bCR%xntw~5`c?_#%tFgZ2Gk2I zG_=_det&$o9u&$yWd~<;J?>`1BH7Waz; z*+%-#{C>Y*eKk(aa)c?n*>QySeb{-^nxLt)7@HeP>PQj}EqqWukQd#>( zC(agiy=s@}@g<98b(eo21H)cwy*N(~r)OuE@1$|i*JFK4Hq8{3-;AeSF}4=EsC6TL zXffq_svhT?mszf6YuUwWw(m|p9B*Okz=SI`En8Oqe$nY=-IpCSx^C&g{c4aZI_$ws z^koqho|Fq%tyqK$nu)Ww>eP%TLhHz){n0ua(`dGr&Gf5pG!wNEnTXiJHX^WuT_RhG z(V(UgE<}T(u{9^+G!_A=-|>bMd`k}!H2g-|IWCtY$sJhyB0{_VZ9_vvZ^wyhjpGN; z@!~+s`9^3EBg8h@5ZuvTHk7@xq3A})4=>r9D@K*_5A-I=Vl~#Dy=oWDgZhrsLwxf} zcIl}^2Q^h|M$=F>(PoGx{oS!-Wu8mK5k@t#u6pUd#C|gb_tcfOO5j-+UTjR z%~+FL?z2_RZfQ*w%whd|85iR5b&Cpllj?c8Ky{9xMQVViHy@HHsxf3#Y+uvb#c!lT%6USv{x#~MKM@ZX|;#sUI^3Zc9 zWM<^L8uwik6xx=BEcOyPj>(vu<+At96$`{J^!#gcb0?Ym=*;W9tTE#<9DK!IzB_u_ zH;vX60t(W#zjMK6qB5v#s-y z2qly=c5!F2I7f$9@T~CQ&2zkhmo8Kzd=o-^|HBso#DhDY%!jxS>eQu*Gn59aIDD0a z3w@jsiv0S!$lPvZF3Zix{_w{wf>-bHideBwO^7+9ww|wMW@NwgP&?y)n$8%VIp)t2 z;yx7P|H=BZ4nyqMU7q1eLh!oV!kv`qA$1AA*HAnP)!p`ta((qPHMlY{;>ms92{G=! z&@BEQZ&l3%&$>!vR8=|tM&%f7*2v}9RyBC*Y&GJEyR`)4ul3e5Z=+YN+vuy7xHIQ& z^6J%Ywke~{do-(t5r^;5X;Qwr`KRXl?XTUG`36^JVX; z!H2p>Jo^ke9P#q_vYeS0Wfpu_I+^jVQWRFLYR&94(ZP+}zIfL=c*B5*$Br4fo&GB+ zX6|-!-?qoA=IrBSsF2~Io*5R{%Zb6k$3FB5KHMtegV%h-Sz7~-nP+|SJeng;{zOM- z>Idoutl^VmSyF5i)*?5PHD?g}9>k?P)e7*i3jVt?zD=QW?Fyo5* zk8xD}HSnq>?!!EbxN>BP{l6MlZdyzJ=Pa&NK4|~Fab+EG<<0I9W$Ve|@M+@8eKM{b z_EHv|DXz?Z&pY^&0TD~SHgc;|T&XbP%Bk7c99NDG4*ra|a$l>61Eqeaiz`3KxN=m+ zl}EG_&Z9D}e7I7^6=Fv4w*5qw&=7mlph5l9yBO26kWYL%h)1WS9!W__10f!Ll9Hw- zCFdliCMD%0^~jOSl9Q8?E)^+O>@EJ~wfkRQT`d+3XZ57$q!em>2I zr_5=XSku^5W``*qI`AU7XR_RNrg(=Hd5CS2E)!4kb=sh@BRb06Ck+`8+o40`_%J31+eQ^*nR!#EYuB+{(y)8zJ|HJ!Ts)Xcox z@y5P$l6uPbX zV3!sU5kx@js32gaiHL=AMnKnGYq|dYt$nWVT>C$JnMj-&OagO^ca%Bj^W3j3!(N>P zBc;+w*Mw9_kr=RzCZS<4Atg@|NbsV$Cae6RVx?mzh`G6 zOuA>YVlb<7rFJW^({fQ+R)@Jl3bDgX#y_jwgtU^_ZY;bg>n?*&n0VK4>V0P0it!e) zw&jDTGwE4prs@-KJ)~ea6n%Sb>ASBkZT@&0>#Fs=`U|*o?RfOW;sYa2XUlbw}grzh*FL@Rs;;;lNJ@`cdJcdtzK-Tpv zfJMO(2-xRu7d`e)bm@5P8NP7QJ@nOn9rxhpffwC&KXBA>+tsFhaqW(qB%QU}tH-O> zY`xgEbj_CX3stK(o!Gy0wcnAzD%TAM9GANK?9;At@lGHub@7bgoAEH*`|V?YeNdbG ziqNsXhM0*vZp(MSyLa3;=+0Wb9WS|Y$cff9I$NKdUthnwb+z`U&SNGqpITfsH{8xr zFy7tbqV8QE%ZL1Q+j)um)m{B{fw!F&tv=_|5Ix@PxX|g8!*Qd)W(O7fqs%zO_^nk+ z)(5rL*X_DxuV9f$Gl~A#WG8Q$L{u=^)nrRCib3%qKHjvEGYI?GSG)73wal`eecY(A z#+CDzZoYSnv9r-iQez!A7CzQsIdAcr^FFmZ8Z1Z)RvlwTec+mlDOzUH7(2KuJk=Nx zHB*9zeeLckhvR9nhJ#_xCvFUfKmT?k=-ac2>%((zuTO*ze}DUm_r1BdZa;PfZ$JIm z_4zlc;7z$Wykl4CZx0K{-ySD_D_rnfnd9-_3QHYkOUHHGNpn5sBjzu9OdK9Yq#$81 zSR@7xL%`q&B#IBskHHEE3JHsd;_w79B1wFngrwAbX&G5LGKDHnQ&3b=R#~u6bboG`k*I!{^NM{%sGfhm*SmqX%Rx7P-Z0+nm3T)lSvMs-bXT|Kv< z@n+Ml=G!f;ciQf@cXZzCy8qzeqsLF4c0YUmqNn#|U;n_X*Kgjwdp|fdJTm%W?Bn>S ziO*lYew&<{o`E6%;T6D(1n}Sm^2j;g*+9k^+Gp7Jp@<6R=4=`5?T+xj8%?Nxd^GX> z!=s7se=(ZS|M+O)|A$8t|KE$IVm5-*galI^;tz{MF-8{}X3r-!|L$=ljM+u3MCTBx z^Rihg$FuB1dpH-a=UTx}y=e`u%8PjR`g#ay*yV;!x#HoT<={mobesE1Vh){RhKXaS zI|iCkP1EB`5vb+l7jY?iy^|RQiy7*VA{e#_1rOg=(`*kEJbYJ8E-|)k8Ly@&oBO>z z+g5EFV6Yxzfdhp?3~cl=R0A%Hrc8WbgJ)Et6tHgzL`z%nF;U)F3&OpXw(J6z=nPaK zHc^3B3U(=CB8)0M_b$M^#Sx8dqYFeB>L>-a)o^9pmtwE&H+h3H@4&zQ)qx-4hZQx5 z&9715pD`RcRGo`{y8TJn4c`qLHZ+Q$q%IMtDz01ouYG-XEiZWZK=5$uDUR=niV*G5 z({~O!ib1kg^2BH>=-rz&LrP8PI)7^fLyY9$`@Nov88?xC4qM_ z;9V9C7Fa#tv2N~=?@^TF>2rsiL`X8UEpR*H{eEwD0WGSeKCL?O@s})$GOy|+5DGtz zZgG@pKzJf1Q9;>!_UPv-sDK%5glZsU(Ml|Ts~)NXA&n?Fu zZ=q*Gz|r_3Z7n2GfhAr~A3WA9h|#dnMiX3`ag&(Z895UVe&Mm86-+jUjmNVtl+4FG zeIf^$Gji1Ac$S6a@sA!UG zjY0~1t`=VrFFwJ{41dbDw@|t1{%+ixwOQkf;p&z8mY-&x$jA7e3eh{UYuN@;n`wWl z>q`BrIpD?Eo-Xo@r7Xg-cT^#}NWc(vC8Z3DlBmAHbMtFrr1pyt9yX|<42?%-#wrq4 z7}!zm5cDFCcc00g93}_Lc@+0KDME}R#vzD_rFv4;+RwNRA2U#|txRA`+*f+REa8Mt z_xH-#BsL}ZMw>B{+4R^*%#mP;XU1u1IcYh%>1k{Z9?KCEN4Ba2ek0~`7AlqoQ^VO&tV+Bdc0bku z$K8b{JmgbH4dW(f7$aW%Fe>@kTLnVHI~@M{TCZD-q_zhXqv!O_sKxWyib}m!3V(0z_@Da>k8{`ZimXwv6N3Psn2IAR7p=jf!yX z$Y>^9O@t3G33n|Cc-ZZ$!|Nr1gon50+I`*02O>H9>8eZ4n({G8pEw-z*X3U{-MJXh zY2?X@2@P_@A-ngA*0DZ9Ul!_fadB^LfC{MCr(Uoal zH@E>p;_lBOafC6BhGjI8ppdA{OpzM(Dqgcy<+#<@!glMkil>Eq1M?eKjJ%&RkMZqa zU?y`SrQP>Q&=X#z4BPa&UT(Hi5|*Xvk>hz0!7^gXsUqqKFe1ANXIWy$iwItg$Pykl ztIbBl`jI#A6h)ebohYTtG&PaeFFbWiYc9$41?nlR=r-9HeAM@Sc*Vz zELSY<@(feVv(DpH^$K*vihhF?Leu(=)%xp3ZoZ6{JA7>)j)GA4Mlq{735X>{OmA;& zj}{Z|n$>l%oYx7frB>{=LJReJCEmI=65DuYzIMcEX&>^SNB!%xkXq% zjksiwT_E_w#tyrQC0)vM9((WD6s$e^-oyhG20D*z>UAY)ET!@FJ(Oc};pQ8Pk79h(jh2tZ0fft1OpmQ5A9y zlf`$K)CL1YuZt3elVN}Oi^p|LESt^7;dD4=;s#7kEE5YxIYI_bT|M2?yY5J&!>5_x z#9Fz-zH;hR1QmgZ-Pv0hFN)7LMl~gu%DQNSqvuT{kAFV;LktK2Tee51v<>Jy*vZUDp4Be3rv4UI^ zleZs9nwg#VAI!{7^$%v^S?1AK(q`zk39NwVvkxP{^ciq6Q+-CNAw=c-dVpzxyGKd& z=EgjJ{8=nJxpB{9u*4{_JoaNHVL!K$=)$R(gB1!WWzMZSB;G2LRm59PfZvqv4VEZ# z^ELJLiYm2otx0V?Py|*C!`!{EO7&$&_v^EpI zKiC(vw!!ABZJT-@xfM&d4J`oS@QpVE+|DW=c;waqCfT6<%ko$(bW=b=k1d@*)i5R7 z!BFgIcJk_y05BIw@)jF}hjF+`Z0(F_%#CI87!qFUJywiNs~)iFktdUJrexf^D-CkN z-6tNJvkDaBCj-XW7*s3wt)5+?m+FlNS`WmtIM7!@f|Dzn=z#|uyr5GIaKhkyj z+v#4{?&5o(LtNhj_8sbtp^GF`#(Ssh;;0(&Of?n)hQN~$>q@-1-H$cDPmgH)>{^0Z zuABEk^fjJ}GbS^wsIpxOQ?;ot47^tE5o6M*Y-*$$UUB5nG>sq#_r{ZRSXTNuuMd`iWmSNYer_0@5Ew6?Tn1P zi@01SL7qkOHE! zU@;!`bd80|sy-pqwJi^PIKCb$e04aB$KR@l>{eR;b!O(lk|$Ny-*v4cwB={LQ3sLY zwy#!`99OS4pUv%|v;DA9CU6Wfj*W&plz9DEHZ;yopS)!Q^41BY0hsO(!K3KcD*`@f zk8`p3>0~n5$XcH~Kw;5XGz;*oL-jIyxhN#|%YyQPD^Ko$DyRct68UqO6f!RG z!=f9+eitT`az`<3AEo0@?JYuNPqi25Fnk*C=kX7p3K=;)x#;@vx9O=a{a`0+i6`K7 z>4i;n0Wa(voodvDI=n!1QC7Z%nX|aPAVufxYcnFXDn@1 z)m75fBS?v3D#pi944G(eb{6mCL@1qrDzF~q*CIYR!L}vpC`UXmsPwSOxHr_9nWOzB zVx@CRcy-3H-PPs9_~9{a^_6=~i0--ii%AunZfeOu)-Hu&_}^A-hrRR$QQm|)$W zUVsB-48A(pS=OWHhP-nz-XzlebEOa&-56LAOG>Un;Z0<{TBo!~k_3L|(UcO(^2^}d ztaxXVfGH#R2|V}{sDZu{Nt3+)6yxtCu6+r%KF+@Ld9qgV$%TMVU_uR_nSSA#eo&$r zOtnaF-Q>qG3|*R~($A@-Q^*+N@hqdQytVj`RfY?Pj50JDnK7?)&tF((;JIC#v-~_1)|&hzQ=U zk3wL;`UAg3O!Vax8_NOfILPCk|C4b?k8r#=7%pm;I9NuXC_KJU!S|YB}_Ygs0*C3WE$mx zYTNiokDdveH-VqbRai(geC^tFGgU3f}O;I}J|zxw6C<<6y3eCrX30 z%5?SEp=$Nh$9qgaM<`v)3cIZp|1R&qAY15Wa>AywH}gjhV?|{)d#h%GWm?s@8^hmj zL>vJnJ69Uf(Fy5Gg%awb9J=5fdlc??DGF{f+x=vnd6xmwy(@V$cDnazFn@2Q#+A*0 zvwdn{`3&e7-u5?lO=}Zht%@%|rO#dfAlrkpUL(7V-~!vVN2t^Wf`hTGaQY_RC4lc; ziK~)b!%rek44b#)?Fs8JS!Y;bUfRvCs=oQdjKw_AKNU}Y@1Of!c>Obx^8MPt+%#U3 zMV8H$W!7#;kA7#!4=N|sl=N8uoDd6uZRo~X0&YJE=Abfr8$bY#gw61_{^3`fejWdR z^j6SZ3`W3aqW6{%UX)j&dp(c@I_`zs)u139+A4R!;0wGt1;q2vU&XUO36=%LGq}4o z_j&*6+e^Rt)Ia(5;#>^QfS3+HMyRQd;0G~{gwd59%A9@;&Hv6_+F#x1KiQ4`?$9hk zdX4z$+Is@Rkq?%Ion2}mz*6J?lA-yxt{Hv||6=$#{KW$#%(&cvkJm3*q!$Em972Cs zGV$ZJ6=0#Z4x)m%&+pMQGt~_HV&YR+H|!Z~10nOkc^?H8J#%wYLdL#bKJt>6Stg2v z^aL#*3RaKBKoim;KCYAda>;DP0DfQ4`E@dHbj|b{@u}N(t(*VolBsqvKE+pSo#ZWD zQ}uX!vd2g7d6zGlEZTriaz5`(I;m@-Vv0|++2t*kdx@!_i07DTdE+^{ObQy$HvH%% z8gj{4W&oFMdUG(x75G%2~PAo3PJ?OF@q&ta$1rvdyJlvT-DC(ONG78<#qK{78(< zQ&02~ms+gpNR;qePlTvT4IDl4eD93Gc&^Q!XPt+~kG<0Gm$V7K(Rpal=hd>VT|VBiotNKpRkt zleF@j?l^G0uYYl1uI1Wi9oZL-^{cLzv~;=AvA@ixe}QAJg~Q>FtmCwPWfMsY+r1r` zhx+;ywfT02J$Za*X;r!Z;HFh!k2_MAoPvAZ!A`Yc4VY#9g`b;ZK@gCo@lLAB zGvv;WRL%V7?+2Ei3u)bCt(n)hWk4e{q{UlAGpG7}|Kgn?w^!fM$S&W~zhG5Jvwf;Y z){*ypikcy}EUYy$_HF5-iiI>8iD;xnyniV>hWR$>F#hGA{RqD?8on*;!NF%^*+JKO zrnUxMJlOrtDdXcTF+NX% z3W60+LBNV_sdOI+D$J8ZgEQ;NaM&sqK617Is{m)lP~fmk7M`vlgomlc5#ZE%92{mz z!;-ax2%vxnr_B~9W64rNSXd4-5tiY_hDEAmVQQ9>h2NZw-7J-RlX7@{MXK=?)t)@` zKmglU^7PBJ#g>P0MhmxmtHP5kXUwfZ8w@8}-8O*perZJ%*)cI`5e zH+`EIcJWU2z!>(4|1ZX{6j9iQfF0k5Fhg~YtK?JIPM8Mlf?sO}N$~zc=Q5Wv`~3o4 zDyxpOQV1l0drCHkbp1QBu$>B)`;}K_5V0NdW+@Vj?6WZK6yrFIFoB4-axBZI_8|mDF6{l#CZcx6E7Oc4B;Fy@PLtDynOX)Vjd)r`^&8( z;PKob{L4u9Ip=-P^g~8pT;KU1wR>6Mo4bcJol~Fc2K3Z~kKIpsvef@kh1)r&l*d{d z+Y46gyr29?WBsi(8BM3;hiYE6ky9VKk{&E}yS%;oENSLXkMh6#&A%&@`u1sTY^-zQ zPe1oRyp7<6ZB5%*-w)1J;ACk;XLw@o^NICWkyRpZbDvFYe7uGbd6C2m2Q|_aGzf=3 z$!7%*3BfiLuKq5ct#(<3-VZwf6Nd%8zgMVfW-YC|?df^1bEYe$w0v8#D4M2L5^5gT zcJ-V!u@qNzKJ8GbVPZj4w#nD>I?FUC(9DvF6b&W|M>gE^@kOPvnTWK`mU{*2#->6N zOV*zEtTZwaASzj9k=2cu7@VB`t``*yW3HTp6M3q$ zt^1OHXSVPY?af^!D>KD~A8Y!y<}6y3E%ZozT?1QKOkC*U5|8V_gB|+?A1qq)zY_B{ zKQ+-bcp#>x!CyZ5?N8r?M@^OBUz~d{IlAx(98aD^7i(m9=lo3&~+2H7q2Cg|x7c6c$p#LPA(b2MftyAr&kT!G1rzf;7O81Q=2PLjqt( z{|m`~A@wgL{)M!^kn|T){zAfENcRiLej(K_B>IIkzmVh?Qv5=KUr6r@$$jV4zPtm4 z#Lke`8In3fN@qyu4C$O9nKPtvhD6Sg#u<`0Lked|;0)=TA$c>TZid9okhU3;Hbcs0 zNZ1VNn!n4Me>=H}zQnthX1JMmE$wdg%uED-Ztg8Buhol;H#a$1&Y$ZXIe*nTND?sO zVqWLqODrud!XbY25rNtM0h{{`aOkWQl8!^laY#50>Bb@1IHVefMB|WV9FmMfig8FV z4(Y`qxj3X2hs5HLRveOwLrQT-C=ThwA(=R&693UZFjz<<4@u-9g*+sXhxGA~JRVZV zL*jTy8xKk2A!R%yjE8jbkSrcj#Y3WaND~i9;vq#mB#4Lf@Q@syr-q*!2p}^6Hi`5KcxPL#Q%`?ACmq<%6~}s59$7A zW&eNbA0B`1AEnuh`AtrzW>-$yq`IGN6lOx#aWEgbWM2O$1+xe7xBkJKc7F5@1QK#q zK*kElR{_~7AXf!ss(?Hdkfj20R6vFb$WH;;DIhlmWTt?;6p)nya#BD>3dlzR*(e|v z1!SUtJQTl8lY)?g1Tv67{t?JN0=Y*Z^9bY}fvh8ta|AMuK)w;kHUhauAkzrs8G$S# zkYfZgj6i-7$Swl8MIf`tZ$T5)2D}pQZf*c(iP=GQ2Hm=$Ys&!KQSQQqQ>DDsTbeWJ z6Yqv3Pe20QbZa7}02QhgMLJDsBXW3XYG*KUqju0d@15&|hHvE0EEe>6XoQHh6I$Bt zF4oq3?!vSmP>QAHZn~-TWOzJ}6XbL*aOG+bu@hUSN>App6M~lBP+WaEZBVcRu= z{C(vd-^D&*EA=(Iz>g>DgAW347S<`FMTs~RKRPzeUKvG}PN+$T(J?BB$de_NPqVA; z*JL_KAqfSZ7za$0>Q2_AW)xSRmL(b|csR$&Mr!CKLRoWPyx@k%@5JS3_2)~d0qt7^ z6?YC7l{0oqw=OJyUSaF7STKR!EKdouyjx(Qvwd06dfLNSj|=fvY(9);aQKI!1Y>Tn zRWC^>aNdy{KSvfM)!F97-M<-oW@n9(*XA>kC6oGU7WDtfL*-ktTZiaKDr3#e%_sx8LF4?vO+P{m_uxSJdky&CS{A zK^r0*f!gfR$+EWddBLBK4y|AL>dL@CzvD`-=BAfjZ^t%uv^QQmbt;d;3Es5~D3kVG z8R*gva~<2{onedl6h%=Xi!@4Tz#H_MT!kWGdbjli>m`n&B2!9wUJS{tgX#{2VVtQ~YhXba^nDqa5|r8Y(#aehuQUFB=kSMn zB`Sxj(#xMpGM@}MCN_$MOvRlz-~2_~ItgQ& z9GPA5BlFxEmcNw4X!8wKlZMYlus&xdg9p_yyb(^AY(p&^**Cor!l}kbHWR3N|>{skrQd`;3(pSN`4D>9xsa2L7AI zIX>aNl54_y%M3=Xjs!dT+;^yQu-OYDRN{%!;r%Piif{R7T%`{D z9EL5#m7rH!$VeAxzd6nmF#&Gvl>Gws}&!5T_l>G3vboz@G zw|@D;a)b6QcM7|dzDleP@7?cwBzU(7sMph5o>bkw@Rh6Cc-mmJ<)+h-;DMLR2weJC zc}B86O@s-ft27WfX=t(j-gYclRW=Ux^SQnHa-nH!z}1rWOcB|I zil-c3eOi^@nbc-HAVzQzsA@9a568FF2xVFxU90!{f!DpjnBo`nHpj3&Z<_1PXMV26 zcA`H9_5m}0ewvE^m4)N-iAH$?Lq<-!gC+(f*G~#|B`m#$+`Y>G+NQ79^2WdJ>pJJW z=E#8{$5onm(c#l~PH{ffuN~V|)a?Sg`&1whwJdMZ20jkS&DIjlUfWnSc3d<&@lNDj zU4FeJm@WOz4g~WUDl(yj(mxl5G#NsTEiRe>H%c?rX4=LA;YzQ}2U$vLQOknGD`HkG zG02mLtuRL@EGS)^TyY;VUBk$VwsIOmDDNvkq?OYUcDC{=hc%pu)0SesUT|Y zviw1Tf^{xQ#c0rQ8+MzX zm8zjvI-Sjnq_~`}Z^6Kg$OV~BnrK~$txk=g!0q>|-WBaZB?i{9+-?%HvfAjC))FTL zWlk+zt$vN!8MVhpF$sJLHYtVP6Rv1k_GatN*{^o5+`x|RRCA@`b>f?^*#{=yw)nWL zb#Y&&q-$TMtE-j4o5~^7p2v+EUo64VW>}EUgKL(F1h5UwnjCzb2+UO1K`Bz@){f3= zg)3qo#W>fkIk2`eqi7g(`R~2qCS2eA3 zZ85LN8LQbY4Z~n;C6v?jZMAS@3@SsHmJ-ir61ANr#i7DB(#0|LOonn|Bf>nbFRI8^ zoQQJOBG3X@RAoVdj!0#&SM+v2x;U%I7MEz`>ZzN$j`97EoBI?0bVC6g9p zDvGUA=;V68)9m-!_@+Pszw61ZCz~y+N?*_yWr)(vF2Oe~+_D8KUoxhUrTI*giQ{CjCUHh;Fcf2+{>I24v#li$Ep zdljW8jNEtJ>yEdS?41rZ-`W?*Q9sq3bHSl|aiSNW&B4b{t4n)(~Fg`R*1}Pz6Q5RX{xt+!@fi)?l3DA@Y0wT%B(p7bdiAYe8jEbqREtFm6&vrXe;^4P@5LEkXOGO^Rhi?Ue<;zh}nx<{g{yupPIU;t}FCtmC?~R%?YL_ z6rLwkRPB?=^*eIo7`zYjZsCi{<;_0dApa^Vtu!B;X(ybl)e4=E@ z!-GZFH~RQbF+ZGbsZcS_x?}n1`o2ZM?a#jKl`*0n`9mjPoca>8U5g;1mE7wYSyN=5 zWl~R5Rn6naSiFohmz5IYbf`7g6?zelyL#3rw;G#nl)@%=Xh|IR%0OG3yX#3zh_{o9 ze+j0A8n1kQY@;hq5J6Lqq^I_(NfEPAMgbDOl4$5A?SL-a2RcKzi(Svu5j+jt|MF7SBJRRu#a!1*_L; zGJqMCF{Q=N>uz1df8p8c)K7=>vM%tQ&Cc9(!R*QPyY7W?=hM#Y@$0DA9O)(X;e3pl z*ajC%jq_+&+NS#EnD>o-JIFg58Rue#{K`Pnms*hOK+~uGvFZ2YuZzIqFRfCXZThQ4 z_7u?c;g8BNeg%plt{Wa^xjqiz4kf}CyuN=&ESzxY5B+_;^i4duQaPCkM-;2wL3Vo9 z7U}1h=m{m9Q&x&nIiX@!t1l71gHWuS&%Y|wRV{1GGqK)0XNRmn&JKd8m6vqAZK{gM z&8$qdetlbA5g1aOKtLhb=wned#8L?wMv#dNNRyxwrC8B^F1D&fOf-T~Zxp4B-SIZ|n+2_rKrnYof#K`U_dKSxV7Ub%4I{kw(rAr&r;>720afV zMnii;Yno9tmC2_{A1)^BhRJ~Elijsy$q25hI5(d!K1Mocm8bfN9`8G6U5G8}E<*KS z;8tZv>m@a)Nyjwgn?yF`$AkZBrDNhz^=d-O{CtJ096H$CUECPqhV5tcq>Hn&ut@&52w{($2yxqU(r{Yo1N+?e= zj@fQrC)9pm7yi6u2i$LGU|`jYE3Gs;8t5$T<{|x`YK-V?@q3{pv z_ZuJ31k0!x?cr*$AFGiElKR%j-|KZUj{J$7YBLDbk68(oq zljz@zCZ6#-ns;%McNYg*t{!4s2SgPU9#Q2xKV)+K4}OT1ieaO|)-$)9E%Z-yanD`Z z*cH+V{Jh<}vKAihTG;@sc0q-pJhYZfD zPH;Z_q$X-cZdJ9eEIcX_7lo<(EFOW2+neO(jj;1-il{eZ(#dQ(oxY`R9ym>8simhy z#$tfCk`qgaOc~`Dypj3(GB;LUH)J|Op9*~cRFpTtvA`45tVmZbE>QOqW~KLj<3`h{hXYjmJW=p>Hsf; zWn&2ds6xQDs?%I|7L<~h!r-uZrsltB_d;H~*Udp7{zTKc_=BdShzxEt@gL`|S*gFS zf0f19=WjC+Ny}2I2Hn6_`O0Ol{CXu%T?L38iL^Se9APHU-h6%zop)Po*lvc1h0psOvlYeO{=g#v!6S65e9#B!Tg13C2YDKZhR{3dZB`AXv z;RNYX95^PcA`hWk!j4e6pDgS4VEd6wybO*v_19SW!3u#e|Rc~tEodWHdtzXwCz)O70+k8FSV9ebdz?Nl2-5a zkjFkqEB#8w<1%11wvKV@ctDWHJRk_LAV0)VP~{*N>_$?MFh#|JWkO#%!%?VN+y_3N zUZKNOQnY5VYETFs^Fs_PKzb=^WLVMjC6-h9rtgc~9%d|thXy%8lO^vlBS*{QYVk~E zJFz&o87_yJHS!=hV#V@_H|k@r%!dw`ZJ8d$d`j>glBd&o0oHdg*!CrGD(A5B)0Fqp z>zZ@yTQBl%Xz}m|bw7^|B*t|#%=^daFbmMB-f|Q_{oI^KATzzAG3*_qhy*x@-fd&0 zU1Iw_Ke?9YHy)epEY|kc;WQXCCIF19Z2{$_BSJ$rS+zb=S-PoZ7d=fmve$M)F$Rnd zFNV&|2NlC3rTkzf@rH*S0w7-yI5d3WuJs4@UfL`33AZ+oz=2)GFgG1iB1`8;j%^#k z;6cDpsdvOgXQuFAkmLcC2rJ+eiaN!PnywAEOBS`k?o&qKDF|xa^3jp!a_KpkQ{;<2 zTO5PRcoucuRQVT#kV8B|NXGApNxzSbft_bB`>b@4)V6KgVqL{2-PK5R82CmfYd`6&oI3KbjAHnSKI z&AxV|B3h2a(Tz=IF;ml&z3Q-g1ZKi~K7YbYg|-;Gt56er|p|!ymw65@t~;bEup@qEhBiIl;SmGvl(&*WaC1 zJlws$dE)(LCo>j1!<&rvMtv{<$ky+RF!jAJlk-De3HbFdVb z@30i4jm!a&!m#%Q?CCJcu@e(`JRm3GFwEFQLy1df+t(jpUAmkW5ogAxGwF0DS&h=T zqG(@s8Y?G<85_%GdeDR!$o0d`_g# zM+_T-XW;e;won`g<#}C!7ZjI%u8TvA{b-oc?+47^Fj~j6%tQT=@r-TlK478@I%QN% zk}Owzc5w7Yp3}W+iJ;(B7x(IZux#i~EDgao^{CzBL9{e98`3ana~K^^3IIf_&wr3t z2fTR_tP?;)$Qd750sHbQn2TYeP#6?Sk#+fbh9awp$&6-31M*oO?4@v1_=CE!SkwU- zytmVm^n%35*11IlSVMS5Uk+y$G3&UoOazV2<9_g-R4e*|P|W>L?NNF{T>`Us7+&AM z^Ya0d7=wU47$?SC8O(8VI!n)Y?mpEv+xJs%VueO`uLLHt=!y4veqZ-|>aw!oEfeA{ z16F;gflnp7K#E5tH>7{a`?MHMpQY`cHFP3IZOv~A-tE5f8sIHH6Dv=d82|c?cli62AAoVs z@pQ=ecASa?=ku5+X$mLr0_MrXTl4*y!pef*nJ0*-=#eUG-{Vi4XGE~~$A zhnVq3lQRKe>%-VA;>1GkXj@hIMKEm+f13bckFDOyqK8Rc5>e73@|$5On#i%N9rd0C zI2&61RC;>wYR3WTx68Ur>T*?@**LMF$<94Bkv>7(2}iHQk_ ze{c>3!zLatkCWevxIV8?>X2@_G9FOl2x8^{92n^_S^LB>=dEqH-}iISK6s zMPRVupg3YVE)VDuX?&-ajdy_x0I>F2t2sCSMOBUISc)ZfHrk%zWO*YYTodF;yw(xjrrYdm6iIw3H6>?p?0dcMh5dB z3l&3ykpYO$uEguL{~&KREtNzP9)4eQGR6lL6bZ{I0GkJJiKy*h(7^0TeDx)kKn-JZ zuY?*!$H13Hka#RLyd4Tq*NmIoc}LY7%dvPN-@X@|PV~mc9vpFAc0*t9xLjNP*Aby` zuv!5Ryd)<9H;N&EK~?b*4&x%p-{o|Qx8mZBD1ULTLE@cj*qhFtYt+o>+!)h|&&Z3P z-!sz<;EU8wi96@r*_$=`4^sxTZa17e!qq%>Eoa%rkB_!I8RA9D^pv^2akHD%dIp8Y z|CLS?;utXvQQ~NNfg0Nn$11V=0mzXBtjakIUrc4fiWVgTp&nUyd5KR&g@rIjQ z(GG|=-C&`q(z!>eODYk~5$ewfUPWko>l?^>&Cd*W8u9{L)6cTF=XTgU;=ykoB+O!6 zhQmO4^9mZ8tDB61AV3WCnwj)A&i&+<}<`LgMSirx^3u{uBMQ~N>dK# zRU%sh=%iZhu2KFo(-Zq2v>kYq#D*c@-Y915CeF=|3fV5GeJoTg0Id4qEMC-4SD$!t z@zwqER|7U}lSfVrdC0CXJZvL(xNsj1fj|HNBJSwBTFb^On|vI2w9{W~m%oj9|A$}x z`VU|KoRd!fV!QlzrB3)~_o?6P0KeESkPGMkO$LsCb{PHeT>Ymqu>9Sj`M-zl@;A5$ zuv;GgMO=iaDNWnR8W{YKTm&WZ@9+bEwfW!4MQHgW7lB3k2qcMrvWD~H;{YZ@{RbxF zNZ8o~p(p+f0mN5rN%QV z8if&&P@bh^2v1ARQZjV^0hA28UzCjh&nOvvHl)O#K>!~J-Sc3_7PSNU1mTCr`HvRo z+3azUeiX_b2c?dKGRHxQsYaEm`4$2t^rHq3z#z6_=pnP#qx;Q9X9F!~$ z$`uEtii0x6L5bp^JaMyW;$|1A&>9Vri9$K@pcHw~X&#gy56X`RrN@J^<3Y*spxk&+ zYCI@29+Vgl%8Li3#e=fqK}qqT6*+XR1j>j9CB*xY4-bxj(g8x*0HI`nP%c0y6(E!e z5K06Hd zc9}=WkmHd#AwouSZRvf+8lO|MD6L=knBRQyzwj}?@G*bH$7E=entsM(mxgsPTi_e_@p;w7l#dH{UwQtDLAj#4{ zLv2fFcW;bOj<_Rjah`vz&yd?kZNJo`A^qd?#S@#?_uyC|G@or)DTg;`>sM!zQ3pp( z-M5`EH*nV&ty!_%>+ZaCzIjJ!0Ww?(IovKg3E)lqnsYZ$-RlM6lb;%!ZI>q$boA8EGKG$arr4Au4cP%^gESR@<lM`opxYWK1D zJ%fWw8g8^OqsCdEOr)kL0j{{!?&F$oQ|;W zD?O|1QhN4QtnDMXh3~s^gXMT1AZWB!hWEODxwk3LN-^`6&kA0FBT_HXxyMri0kcqH z;*e1^3Jn=;-;9quUeePycvI>JJ_ECb#~-najfhtpwW8PdA*)_|n@|d_Reeys!u9j% z!H6Y|fIw z>lvO+sy{UUQC!by$4+@Ibo59>K=i({QyXSZ@wAlr&ZOR-EAtL<1s0}}C^P>%#3dxZ z8|6r{qF%kKQ>aJx$hX$4LVlX>er^7;IhH{ysLZgjP0v~x_pap`CgcHy6i+sUEO_?DV{2$&KH;6s zX-Bu@DIO34kZ}G_?2Hcfe3?j-jmWFSjWpWvZ;vz5^d5G1cON{7T6N9gYem{^h@O%D z6Fp-$_mA`p<2ibUrEFr+5l@<+Sq!^$J0_uDnv0b{3Bg{eBB=I=$he>A8Ab(j@C*#C zSEe>BJ>MkPY;7ny%w09qx+0i>cH8n+gPL>ksn*QR_`Sr{s5K_9E30lJUT(YTuy~(T zkc{&-*X@&o6$Vx_Zid>IUHaBU(S3pwed8@mqg-M~G>zWu66|QdMk=kqN;Ms?f3!Hz z!^`DT@nxq=b<|m!hVzil>4wlf(|dUq$XFTD*`H-3M(A*m&@o<|hZz|j3NnGk-7TGQ z?r3&@oFV;QZ;@pEleZ!)g^bV3gP*mw0y5C1p@{))j-kb2@VX?J2hX@FXV1wWaEviq zkiLUO<{EizNYF-{QitIrHcO&5IAOk%GtQE~Y&R<08aa=cEg3`jf>F7TjkzMT%9eVB z(y+|S)_M7TZJ{jnyw@9V=N-CbCH~=-RrZJ6mJa{Qf?GvbQv>I|WSXs%x&wult1q*t zY$TPhdpdI6WStZ(X^)0H_}Z+(jx&z-z=DHqvLM|6*D-VcXstemFB5?|P1q{`-c|p@ zS{-opsBn>lNc;OO+tRt&vigGv@$^jC0`UBu=k-%M4Qo`VYkjo9lNM#p%<0sXu%_Yd zSo->TD6tS1PwPxiiPWfgJrSgA%1eEz0WB=wfPNl-W9>$X_)fK(gCB1B- z{l=TSp*pJ*a^s7Qlg^(QcYmz2Y0&Srpzxto`t5jO0dZ~Qv$rsq(eyK{PZ7s61D$qM zgV}SF(Ga_aH=Y7F{k6Yx(;pXsiC=b{bJK^x73jf7{XBLrjIKS{7q^jL$ZnZUW4Vs0 zIr@Ci^#0eG$6%_!PygM=yc9Uh3)K4QIXlFK`0BC}5XJeJI29Z=vC%GP2bpT6Cs1!I zBE3o$Exk%gAl}PP>SoqamfG+F^zZ^V;gPU9 zmM)pb=_{Z|7rdx}ovk73ZKp@l2$-W$g)b`Z*08kHq$N0!$+UE7T~!>q&NEG#hQXeV z1lwm>G=6M!U4S$e@weNx%KzWMXgtyW_gwV=qfudLpLWk>DY9e98v?z;xkcjCx{%VU z4(+PqJMC4ML|!#qc5!}ua+&3U@kes!NtD%FbL%Us9J?;WmYB$0MTWos$iI=_@b0Fd zl5UyT;H~uQ8ph+EnR%IW%(G!~YiP*F%fC3@r4s(kOTk2w zQNd~n{BYCg;~~Gi52Ql^xGcC~ENdev4&l&e%LvbeTG4J)oHT<;>S zCAwiFG3R(l0br#d2?WzR23-$r)~ie>+Dhd6q^sdX`X%l}#ilD~G1aBH=F-MRs>-P* zy|g4B1|y$qE*{Uz^c;{;M0hx*&S zy()UBgj%zZ#VC}W&tGSgs0ZLUpu4a7G}Qg%>ICm4Xn6MJ8E29Gle$Gjzi}?M`p0rD!cRNsZdxUZq0spW) z8F^hNlPQ)$6V}x`FI8l`aIU3!kd&WV8bDHtbsgu}IAI(l8Ko`6J^gj?D|g)k>Jbv( zu2PcG!Xo?1yEzN$Kkj3r3V%;YS<@IG4M&l6;{oIZjpWqnn(7G(n|kGA6%nyaj&89R zpix7PW*c&*j2Eqb4_Uq-FIs;g@C*y(ut z(76g+*D}AWBa2y28b(s%iwFDsI+m^yukLb+Mk6$PFs|ia51n5S2HP_D#ugcS>g8Pl z(Ddvjq$}XW_{JZ38C-!wfS1v)FF(i2c(@)nZ*%smUgJxR<)2EGzVfB$2wHDXhR2k8 z-V_M{uncK1rZ1|O(4^Z|+||vGi$T1sp!pV7(q2`NX|OvP%D8+5ul#6}fb7_S>^^CM z>^`mRK2;3%rIt{#I!Z{uyHL4+mKwW5T3VpKha4a+orqM&=%yiEm2nnEdT6pB4Q?kS zU}Ub$M!J%*iLp`YivN<7@h?YY^<;JA|2YKsq8^t}`ULEorjy}3P6qk4Kt`hl1p}VZ zINs9;H~R)fi3M*oP!wfP`x6nH;Px9coMvJd7_L0NH?P-KaBMk^yv;2uT)Z&K?Sb0kP@aCI*K3v4Ni30DH~Azn9JC%LCh(8NMGS{Bs*5 zgrq5NOC=r_iGlt}j`9?oe|SiK#e*OAOey;ytb@HoD{!X9# z6(_&qh90R}Vko?Jy5nzu9aka5+Pv4hmRRhB8*GfAsZ=}Hv3DYpYEmT@B9n1?!4pi+ zNQH>WUsvm-jKdh4whkqX<5#|S;w^E!(I@kU-~4#?>|qUS zW^AS@b3#FQx|O0kYKyW7Z?S<7cqebkA)Z+*s#cwO7a_dkU1~>*Y zRq0!{hAP7kNMqi0vIXA9^z*MI zuawc2ynKKY zr_X()%V>nb5@AX^Q0Zo~D9A|h$_$6dR0M@7?f1I6C+t^px+m*Ny;V;7N*;$aFW8wbog-2rEoG8C#f*yE$5CMsHB2} zlaNW-WYQKlyOb18CT)|#$)qh(IGJjTluc4LaWF8GJ>o5KgeY%_Qf4AdsREQUk5nc? zLCJA9x(}H-Apx2}n^dBsH6f!(Wk<2eISv*`hO-#|$dkwtyctMO0^^H=pmZMAT&B@)EpT zTtr<=oL$D_pup(lw|Fbc;W^m%1AGj58B7eVF7U2iesZmWk`TlgNZ0nCT$|Ehh_iB6})CK#Z z=v?{3)PP@H{;9_6zd+T74r}~?TQ=>v-)Q`bi!>$1L%-AbW-4+1I1yxz@tnmR@2fZ>lpz&kqDs;iVs~X=6681ju4~_St0E|2Gp~m}E zQ#W@l?_Yiazy9U>!SQj?eI4i0h-1ry#@|m9F>q}9iO%C`V(>0-d|u~Q(?l;UHnv~q z*DP`N%gkHaD(KfPTH&YGh{0x^&_rq-krbKaz;6`^n{>1*(neeFDV?{NviKSL#EZn+ z?sN3!b_uxHpHwb18F@>WPy)ZvcXWv$V6O8LG(lOkxTRm02m|U)UeS3Ym_cCgpw6$L z0PN6yRp+B1(SX+Xb%{41?@|mRU?^Cp?>fCXTVk*8UrGmn@7x65sE=HUzLzD+0KEjn z05=6aq@V#hbry8XE{VRj-G&Xnp63k*e?v_&la#4HV%!pcjV~F5I@t2nR}9{)@YnO2 zk%YhForc0+(Ix*@#=?7n~8aPW66YU0Zs(w3Xg zFOM(pUVahydqLvw%x;qpD*QE=d>~DXg5$oQoBTqWxC;CYyl4^>`(Uw4zcTq1OPnYC zbrSaQYs_Ta^j=%SJg%CGs_@njpM#7X-J@^W#CQe>vo^f5Hixar!iKu+jzoj7g^)rNC2)BVMA~m**C8)PGylG2_n@XPb#51+w8e5s_P;J#@L9;egS;KYiTH^<=C8o!WY^kZ` zx~*t965bkq%28fKzwKS%ZSk=zv^WR8irUE0TiN2`k95(V>vQ`(?g{(RcTeIedcksU zkqbYBe{f*j&HVFSm|&z?L~-%F#Y@DAmrUNHF%p!a1W6kzQOAEdPjnReI&=L6F)jDA z^Ou>&QRTls2jrMGoB73|pJjVt-k7#`F6?{k&lYbn#NJ<<{LP|d13Q?syYoFw4BqD( zi}(vzF3sY4Xn^UkMURQ-_lc|OwB8im^x0vpt?qsly1PV(eUB+2p~F>;gb{racZt3 z?FJy$U$FRkarwgp5Oo`fbm=K!Ls3nf(0sVV`aN>aT%{0_bmTBMlbg~9<_kpIKTTem z+z09^R?HWRLLctZ{sLSU_JyLu?2N2VR|?$sI~JcS>fm)jwS}? z`EZl>7sU0c-3cp;17%>0vuw}3V0{FvOGO)4@0iNkBoCrEkM|hjT9yxE0?FB)vX>L> zp5*y^rv(|1bmnb6S=2a09oEFr>n7igkN706_@&Y_NpF;9dm+die^h+*t}gcH_%Lc9 z+vH+)-GeE2N5U*f7_B4|jX&rux~Ls9Kk|-0h$bEUi^cb7qGinF{TdTT-&h2<<#UVv zX51@p>(Ykr^I>E`PR*Vu2bOHNMp&Af>*#Ro25N8`k!bS9|VVi+35K@82`>Y8V&I$gTDCS-e^teABvvTWkKJ z4?`x~t=(FvMG~hQagGbo6K>IA+=i99YYsVaX^iVv(``;EN3 z{3q|v%Ug=GD~n`FC`$*FsD=0it2eDGt(C{c>rsALT|!=oEH4u!kD8(=G#U$u4%U{J zl~ivaR;#K0{gqDrUD*p`WTZfEw>trKW$#asFlx+H{ z?<|3?&r-%RDxX()hxSSRp59cpv!;T02cJcqr5mU#>1c&g>X!PIL%(tp|8dbJ+{?WLJl_|M2N>?E@9n)enTW nZ{Jpzy(7H6X3MsxvKKE}n7=UJ+k(FYr&-xrJiE6xP_pPh&J|s` literal 0 HcmV?d00001 diff --git a/matrix-parquet/src/test/resources/optimized/cdc-0-index-1.parquet b/matrix-parquet/src/test/resources/optimized/cdc-0-index-1.parquet new file mode 100644 index 0000000000000000000000000000000000000000..90aec545dffdc24073adf61a19bccd95e0b976ed GIT binary patch literal 117517 zcmeF)2YeId|G@F2y=$Aar4>n}QJ^ZVTBYnlk*(}0vSp7l%igPODFPxQ$Q0Sdg`l8- zECmEvA|QKDfh!wX;s3cLce%7_NK5he^Y^c>d&j4j%dOn=eZJ3g*HXJ&oq{Gm)9A9M z>ETUGLjz1Y`-`7R_l!-50^Iv@B#fCxYMBLD^%5r`lJBO5{xif51= z&msp*FylGoL>Mf19xoslaw8AI@giQr%gBoeMB){^iq{Z@*YO7OAsYEn00mJ9g;4}W z5rbkVj#!kyn<$A=D2*~Gi*hKBx9~P9pdu=vGOC~|s-Ze+peAaeHr_!UyoMwcn?j`6wS~aEzlCJ&>C&f7IA2Y_UM3)=!DLQM;CNOH*`l2^h7W8Mj!M=KlH}{ z48$M|#thqhwrfhEAaz<#44=D8vKN{ zScmo4fQ|SWzhDzKV+(%8R&2v>*p408iCx%@-?0aOU@!JzKmNo49K<0U#t|IFF&xJU zoWv=d#u=Q&Ih@A@T*P0vgv+=B3CXyMYq*XZxQSc1jXU@ocX1E*@c<8D#iQVWYj*{` z-tbIT_2+cFg||@w6;TP5Q3X{|4b@QtHBk$-@ebMDh9Ch$F$}{o z0wXaBqwzk*AQ5Bn0mflGCg4L%#7CHfk1-jaU<#(x z01L4Qi;;vS_yS+zD}0S_uoT~78J6Qae2*1ai68JIR$(>P;3urbI;_VAY{bv_1)H!L zTktEkVjF(LcI?1T?80vRjy?DTd$AAu@h1-8AP(U$j^HSc;W$p6P_s|4Q(G1Pe z0xi)Bt^g&tJ{A7e5;!4yozr!*QIzNu0uIoWWU~!+Bi5 zMf`V2o4AGBxP!lO7x!=<5AYCHJPHmFbiQXF30YkQ&PI8>g||@w z6;TP5Q3X{|4b@QtHBk$-@ebMDh9Ch$F$}{o0wXaBqwzk*AQ5Bn z0mflGCg4L%#7CHfk1-jaU<#(x01L4Qi;;vS_yS+z zD}0S_uoT~78J6Qae2*1ai68JIR$(>P;3urbI;_VAY{bv_1)H!LTktEkVjF(LcI?1T z?80vRjy?DTd$AAu@h1-8AP(U$j^HSc;W$p%t{`UR3Rw{# zD+2t#ivT69q9a#u{Qt4Jf?rmykQFJiBE|o^Na4&C{Iha}tO$@50sh}bfRa{!N3P)b zzezAndfW7!Slv|Gq|2_;CB9U03`?|b8TWHizJmE$wrbtRD(ECZ7dB2;$U08Plxv)> zuv#jjmWryS7`0SPEfrTwv6Q57I&qvXaoS~F0qb}jQ#7vwL}`GSG@z8lI#84bib(@= zjkgXGr9ooSAWDNpX|R|yn9>kY8X_hQp_Cv>31U(LrJI6uQ%Jl?>6XA0eM?|(Gqu>; zDD^g)dmE?SV$H)vrs(0Kwpf^1(r__pcx0Op`ym>}H7cwtXkG6kt@o+2o~-PEo{5J- zd`zMC7yCK)wPx3+MYQV0JmF)?jg5UZY18z91-4cB;-})5cDxxdB1g~Kbv`J1r{uVg z>ksU{e@5TXU&7DLnQ!hS_0M^B{_Rs=7E88$tV+K9M{d|Y)~M^Ta-r@hy3o1xC# z(3#5Go=pi&b0_4T^U)=b&}*)ZBCgj)2EP5?m4$-Nub&g8TnSuWO3`{%kgf`eSDO~G zUK6BiLgF>P;_HHRT}Zr6>4qTP5E5^s;{p&0Ij)OYX?>6Gzg_op3v-Q0w}ix7k%fcp z7kD&PWT+?&6_bX>=2R}wuyM*oOUZE|Q*vC~lpGg4B}XNo)L)ePi%I<{-7_*p^O_u@ zP8}&bJ{1)mtgMO?;jAa#ohEgS=`HL@4$)lx)h=Hre)5Vb#>b#SOU#Zm5JWQykX$u^SW;~iOB=kOHM1m^TGNz!xyt`5AZZvn7kMN2`x?(Y3u~58hkuF=zm*wJpBU3c5C-Qihq!W3}EZ{gq?T$=h zFNTTIFfnOZsYPlevLAj7$KJvswXm35SYj7hcjfnHiea%-`!a<&sZQPmcPAjOb2vd| z*;pUbulAslO^$6kQwdv82~RVqh*sYRBefBwxNAhwv=c>hrWGj)sgXhxG;~FrbBj8Q zG0tK!wWyrRS+~?vsky4XDQ2jZaR!cwf!+p=M@H$9(fmlhvJ8Hb!EdX9Qh=Wn;I}n^ zlE0ti@3+;TlAoXC=eO05lIX`2&8sGIWL9}0>r^lNQqQWjpF&6K!9{~~(O|x4;2QnQ zz!c4EChdip^|Uuh?M>$Prn`AkZaCg(jW5b7-0qCU3oQgw!~~+&F@Bs4j{fh*GK5fCOWg#LYe@QYkUR+g08qdHsl&F6mpxLsArAf zpq#*r6bq4~##-A8OtadvvySXhH|8a3iY?KWbBSO-DcEmoFu6pKpA_V`HHgz2=qCmG zZ4IPk^plK!TjlHRak&H+Rdj!mB5C3 zV&wLaV)zdm9c2(SG9qMU*d@7*{0dl;p z;VaefZCiu7?dVTXa&&kpITAxk-uEZe7~p7NqVfkT{SE4SW^}rryms5yK%-#w4U~KX z&Ax$=F9cbA0wte7vyU9|^nsE-(5#n>%9Tn{Nkk}W67W>S{E1m81V93sW?& z>DiSoEbKVG$ST>ir$+ARC~N2VG)FFn^>YU4oWXof&g@!Sq}CR5YYUf;k{MA{(Y)dzJ9J5NTC9*;YBl~k&cj|drVX!<~T+jbYs3B?o1WZJ?x;1v9=VXmcpo( zd;y-i547=)kP&&$$PbP|3chO3i9@E2K$M_`P}{5&4O6P-By|Im-Z?{9XGfFN(PZvu zvW>7f$2a7rP;G)Ub<L2%fb}Rt0sBqNZHW@XzP$X+TdtwsPE1JSI5Pst?inXX;=K%@*X~z zDpF5j&dDh0WR&@2lstx{3H!G1KAKl0=#w04g1!A$H+A`!QE67o{@j$LXcy?)18A zdK_^gY&`2xlFm#mdX5hF2n3E%naCo+UJuJCnKjoTD|mv*-av6a)wC!@rhe8Soi&)x z8lv)MSF?6b?-OiN``MRRkE5Az(@{~!Vbaqj(e_{5_Ks$=&6XCarN!LR64g3Yr%V}& zp^CP(NNp|VwsO}b&LYKG%yDwDuaPO5SJ!?CKTcwf;~I;so=wSl_>S(Gqcq)6L>!~-jS4tN+igKp9+P#9u-*OA%Mla#R!pjR>~@Zv4J-ZMyRW4T)9yyx`Gn&> z&EIYk6pM!hC6DWUn*XM!0-S@C?ox^;ekwr8F#Fd`PXeM{Rvpxlp9)aoRb~(Aq`oli zbM|m~I$~)g=~as8;5a+f6(x42wVw3OmxpgOiG#g7fjP<*8@WC>z3cS6d^5J#%_Ma* znY)?X4AwNaNX;$g=JGw!ObthFZ-~lwgq2Q+d|yNp@Bh(pnpPI6mBrl3!WXO=r>Pqy z)r~UOjUpJTUwf)xnD#hLlLA)PDXvKY!pd~+-;A}7&DtJq(U~0gZ<5mvEWWv2pC~D+ z89d=>qqLqPay&a!uRuoc2`kT!46d1W&St+kv%$x@!Kdj4ZGCojGtxO%IoW*7T~Qs= zkvqHU-Jf(1=~y2crH4lIL%Cz(X+%d`J>|}XV;m=HHTBZ|#0}R+dV{OmhAQ>UtbrUJ zBa8h1G?0_(#?A!hC~d2pZKXAQLxyX%jn00@xXMm9X-iHIX-k=hE{~VQYQl>y(KIfj z;Uhf*GHPb-8Z6ue0QX@bJ#(EQMEgvMa_i)=caXgs@$uXYd;e}}xxsea%ZMCjPMMpW zH%R9V=JWDB1|_E_An}>Zm@KTs5gg^Q98IPoKe?t3ab$En)6>mWrA>}%<+5FQL|OB| zjC!9n{kt$ctr4Au5mLhlbHfM^56tZJ^w@^_L`Ml0^RAB)!(z&qj++N&hCHu*V8+qZ z(QbL%4T*Ohqc{C4TW^`9TPE`@xo>vU#1zeIs(rH$qs%2k+3Ri?dG#)D=Risn`6`nv`gb*|d|;IWdXZXYXg z^m#dhPjpLJd8~vRiO$DLlp)>JwRSf3&avgD8c~c_4$h>s?y71AZ`?gbs0`j^ZkI{^=9M!oKY*a=l*+f&m1s}kR{|mj_XMY1`&n`c zWw=KB<;y)&HR`AFXv@M~`RV4YA`Lm@CJixrNJA2ax;}zZ%r%BM9yiPE=T7K8so_M8 z2F@6wy@!13wI}ItIPMuzQ=Eh9n!6*)6R3<*Do>q!9;7VFH|y@Hlc&`0(6$V7X(g1? zqP-O4hFGao@)HV5UE}bS8c6ZUF30P9+%YN>MYv2+0n~BRC)-GheeA;8Zdjxn7V`~v zPn9U+GK4-QE$1*bx0E%HvMaa2Sm1HP8S)UW<4CppBi~bYH1||Y&>} z)GjQn`HnGUCi1(#T7k5#e!9myA7EtaU)6;O*m(UJJ5QU@t{D=w4 z-Q$x6?QJ1tI6{v685@mY{6_sn3r%XIy$a=roYX5$Qwg8mldIe?i6p+LH;gGOlRO5a zHbJQ}SZ(A=Z<*}Z?JZ)ovSd?@u%6v?|4SY08IvQm@Di<9$A1R%Skj_Pz&&juu4@F6bqRfZn_fws! z>CAC7UQyHYDWB(x_4eka8nkwL>UJI4spK8&UGfh5@M?}U^A2BYsQ!`bb0ymGRpLbk z!$I$wYB-?EdU}||aUyt_#CdYvWeZB4sR;)q9{Vr}G0wLB1!?_8sPh4()D4U#B2ZMm z2=_&8$jCrh{yjCvQ~L5im*&wd%}A}%GH{(`Qj??ZX=%2f)`RQHAWLNFXSFS7y2=~V zYG#!u&;IEV7A1d4@eoN`BPkzk~AcsyKQK1XV{)M#)`=!ClDp^f-?Dnx4+N zN`GKldp@Wn5}G0nwUiuUNe+o!M2_V6?H=tVF`7%lF$O~3o&M36v`1Ie+OI9hH)WLv z9O@cUb~|`8vnP4D<$+0hU@|{2c^+=b)GfDSZLT9M{HPsdd6ICqEv<0p+NGd}A@?d+ z_z5O{1vtCA&CTWj)e#iRn%q?$K~WyWr)#Bo2Bv$A_m1!9I+^YH&GI$P#R@W7UfVLZ{clJKt)tSWmG{`R6}*tKuy#_ZM=gzco%h1 z5B1Ri4bcdV@gAC>DVm`Gd_j3G$CPz=LxjKD~Y!f3pYF-XK%e1LHnj|un?6Y&uy;bTn3Czyh%_!QGH z9WyW!voITpfzIEhm@jWallb2yI+xM=Vf{?cuV(_P|+%eVpw$+(JZxQ-jRiCeghJNO%SaS!+L z01sisBj^nNR^5MW+N1NghpEYFg{hqKU(FqAi&I(G>A&GRt#IyXU8h+M74@||-oo3c zfQqPu%BX^>sD|pOftsj=+IRr+Fc5<<7(!K-);QFtA1ARqYqSi1ZufPyH5 z!YG2Gh(R$FM=VOAdn1jzT7xOS53$PH2uoy{Lf-mqTzQWh|221fRmSH)*!}nN$mG}WaVii_n z4SvE}tiyV2z()LxU$6?rvPT~|!;|$K?9M0ncF5)j-!ev~6gk)UBHC)FH+{7*1#vS~PySRt@cz}nn;*s8u zfkBaft4d^OaFg!eO3)X4@fA-yFX&^%0{`aY>ktwC&+WdTA0;k+^34hQsbc5HuSU>+ zE|z@!VxGR<|Kxju{`ZJ*x%U724q?F!LFdD6`-s7UkME0-_8;UKt^^;W809)E>l=@^kC{YU#Wc12mI;Pat)%==vu zd}fH1H|YeQSz?g)pHepVr2iLuR*TgW+z)9Mj~%1im`MgEa{z20B&^%4EVX9ZtD49e;kkTX?8B~(Tg zR7EvZM-9|OE!4(4sDpP=7xhpd4bTvc&=~Kb37VoAnxh3;q7_=B4ca0O?a&?_&=H-` z8S&_XuIPsD=z*T-h2H3czUYVk7=VEoguxhs1PsM6495tJ#3+o$`xt{njKv2Shw+$z z4>1uRVG=&ZWPE}tn2Jv^4bw3LGcgOZ@fqgebIip&%*O&O#3C$45|-c#e2K5{HNL@8 ze2ZmRj_>e2R$wK5z>ipk)mVd{uommE9viR`KjRl{!e(s2uh@!h_zl~!13R$`yYV~r z;1BG@KJ3SzIDmsVgu^(3qd11+IDwNmh0{2Lvp9$IxPXiJ3zu*iS0Eu7S8)y3aRWDT z3%79xf8#Fh;XWSVA*_)51%li!5QuFmF-#CdeIDlw?VAL6A;^{`sLQ*)oz}%$~755rlzaPM3o7qS2ZheBdKcec>J2XL*#EBVOeG$>tF%WGX1JTwo zC}6sohg$^!@~r~-#c``3KtD96tz+OPcMSaGj)9*KnSw6_h;$Mn1V4W|2?6|QfDwTR zLNKx+1fh5a+3_rLzyvd%Lr#Rjg6Hu9av?YJARI5^CA^Hhh(IJ>!K-);QFtA1ARnTU z9|celg-{qpP!usJhT@1t3A~AtD237}gR&@x@^}kxqXH_T5-Ot#s-haIqXufC7HZ=i z)WN%`i+ZS!255*zXpHyJ1WnNl&Cvoa(F(2625k|Cc4&_d=!j0}jCgcGS9C*n^gvJa zLT~gzU-UzN48TAP!e9(R0)}E3hGPUqViZQ>eT+dO#^M8v!+1==3@aCVi6W22}|$=zQkAf8sA_kzQr;u$9MQ1E3gtj z;76>&YOKLeSc`R7j}6#}pYaPeVKcViS8T;L{D$q=ft}ce-S{1Q@CWu{ANJ!<9Kb;w z!eJc2Q5?f@oWMz(!fBkrS)9XpT);*Ag-f`ME0Bem1H>5n+k$_fEh%-oWLj(Z&&(J6VYsg2uBO~{$_)tqceS4=;&00dc(#TA{B1cwS<_5k_c=lM3%dF%cKwxu z=n(KPo{j~`iO%S(u0E(v&%1zoIO-OG(0G^rBjN zLoF3kOXby44YkxzEwxrlUDQ&4wKP&KeW;eEtEB~MX{lOTt(G>crQg-k5w&z)EnQbj z4{fEfD}%;1_xq2(mTyiwA-4yUa=L2|#IZ}3%nkh?X$@>`nvwRv^%t0Zx$*?0)f@!! zp4=p4LkQAp4$Qpw9KvA1^LPQdkQ;drju#;}33=hEIe3Hja)3HT5bF$o`IGCsi+OvR^|hUu7rS(uH_FbAJwF6LoA7C>$i79$Bu@CCla zSNIy=U@7D#VL86T_gI0I_yIp+6;@*n)?yvjV*@tgXZ(Uq*bKQz*otlV4coB;JFyG9 z@jK)uVK4S!KmNo49K<0U#t|IFah$+OoWg0G!C9Qcd0g8%9H019KWOFnH!DpPqWLqF} zTL#_K4x77Vw8aT<{N6`V)3F>h5&@=W_M17M@>@dBaHDVuorWC$9c{$QFxS}b!)4?P z5q|JT01Pl95J3n=HiY6CWXH3}0Tax44mlBq=kWq^Avf|M953P}yo|hv#4C6euOSMr z;|=6PG`MKfO!bTLUU9^t1WKY5N}~+Qq8!TOExe5isD#R>f~u&7>ZpO5sD;|7gLhFE z^-v!T&=8H#81JDenxQ#bpe0(NHQJyps78;C?cG>Lj_*Cu3%$_?ebEp7F#rQG2t$y7 zp%{kY7=e)(h0%B)i5QCyFb?A}0Uu%_KEfnS#wVD9srVGrFdZ{66SMFc=HPS8#XQW% z0xZNLEXERifiLkDzQ#9Lif^$D%ke!{U?qOQk64A(Sc9Li7VEJA8}T!K!6t0R7W|5> z_zl~!13R$`yYV~r;1BG@e*B38IEX_yj3YRTV>phJIEB+VgR?k?^SFSE_W1sX*MCIf z4j=j(cX1E*@c<8D#Utnp{&KD_7)*{(9ioOti&~1Xm8dAil|;y~@SEx)71h!^YN?4@ zYNwWZsHMSbX^dL>SS`&~OG#?!JGHb)+Rm+I~{8B`IDJs9j z$S=j@m*VnEtUg2+6!yNZK#5SDn3yYE9Gfdt`~c%H9ux3kuFw(_b-F^fmxx!x3OI_+ znQlsj#fhMlTW4ieEVc{9EW92nXV@>uoumyRK#+RBZujIpA;r6Y{JE44fMTE%e#AzJeW<5v1 zN965(SZZa7EqZ>iZL9P|T||M{nthJ=F(;0RPAOG!$YH-BdBq{|z5`b1hY$9R z2n$zp9RiMq!-V`HC_Pq*W<>RGD9U!NDt7ZK$H zJ?zKpD%&}#aKnC%rn4f>wC!tqML)cY?+{isuGIVTd78j(*e?<`&iPZ%6+bRvvFWkX zCOE>$Uelz|H{kwdg?kke}a zQ28UAw6LYt%YJJw`-z{$Hj9X6-?+?1ad~b2K_6C0UN>b$srUe|OKeQ2j(?UwE*9j46D$-eTq0a9xUIibSY2Sz(8pI< z9*-GQYO4IDDA7W`20T`{mHMwa(;2C4eDBfrE5fS^H%h()s4=66H1ee&*VL(D)#R!t zjS`!o8zL)J>w#Jq9oMiePEiNv@Uv?jWQKeO zoK>&F+qpt3R6s?7tETF4yd{)+w^F_?9rex;pS-MhtL!lFDv&aeVI#`-- zw`SNP`P&^!E95?P**Mr&J{nuAMkS|A?W}OirG<2<*fenL@YquAxS*zabH#{c105Yb52=!a6U2sr0-6 zrG>Qqpf61gZLBtq#i&2jeCmkWF3yqLRw^D*iS^wsD$4nmeCoNzY~fF78Y|QayCa|X z)Rk0So&0Tj%Gr36gw2*ixYkYEnUlXW+Z9;)h~I7d8A{cJlvj>g^pvx47Rt{Hu-EpT zZfpC~zDugqcAsO~OueWhQ+HuX~GN3FLGN?5No02Y~=7|-8bg; zT%8lBZR$(glG3K)XMB&dkw#WdMGf&(QI)OBUk@2DU-1h4lok50IqW3Yk(~0|Q`eOB zRcXU)`Rf+Gd$}Gt+st_14)I~N2XIL^b5|C!wVPpE6Up~&Rh+NCtmX?C zKEyY|5>}S?B3@8BD-~|LWVA}P)%26fvzv)-E3YGsAFg~c^0(`$0(;--S{otKg~@2A z%O1&c&kImHTovl2?r=4_ZSCS4*TuI&?XV8=#im@z_Fh+=cwc)SKgDeXc-mRK5&=@= z@x>hNeaf8OwbnZ`eAhZ{OWIAly-eC0kTskqb@0$sa%sr*T50%Qvsk5bq`8#1QoL*@ zmBjW*n#w26Yu&e#a*OBag4?QPSx$kYS}OfYI?^HJx8zs(QdLW(1*E2^t*G1$U(301 z-@8mZ#dY)tv=xxOV}I7#E6*%@Z`?8Vp`^KTdv5QMdo%p;1idj^Lh<-&DA_7|%4%q} zjeHQjx;erO&BvzFMDN3X5zi|lARH=bUu{R)x9J^fv10^;n4U(er=EB|Y|rY4<}w`- z>8<;ia@zm+SykJBobO|9e)D`$l+TQ8ntpbnSX&bv%D>NZ&OVfuX1m5J(lsD+ zU5(m~W{oC|7EGIHS|h}WTwzyT8Z^hKiNM)^p}I*h>3C8t=nWr6FLr)wdtF8(UK43r z65ho0;9bWfZK0+qCewZ+cc2nFAC5a5b2x5A{POr2J7Ny6IJjbb{EYa+oev*av13L2 zii5WIm&e=QmuJTv-p=g!Qe~eWn*cPy_y|8U9akjN=wy(v$cb#qTAGGbAyi=^)w)KOZmnX=pIUsMH z{gyW~!?r+S+fmtO$FJC7TgwdF=bpE1eZ{x7V_bf4MFNY**>}&j^%c`>YgukvOKaO& z*l&4f?OX3`KStYHVwj!4;`YsSwykBeZTEKAc5j;FBzQW*36DR;wzZ^nh7%rtianm; zg!spvVvlFo@%?!^#U9Uay~m$o#~F@ExO{(Y@<7}_wl?`qrEq`k%H+%P9J#=o-ztSG z{P3MTM=r4Ra;3oMcRE)jzu{ac`Lbhac_Yi!t=(U{Wo?3OyY{X8p>FQV{hjvh+CKND zW=*`I*_GwCowKi;H`J}9;;PQFm>(}G?^>}&{k~87@^>}_gp56p6<3q;JugB9{PZO#<#fS8q-)xJ0 z1M~??ZtHc{vM!UZ#-s$jE{cqSU%^>O#^CPPkHZ6W0XmVtRT>6P_s|4Q(G1Pe0xi)Bt^g&tJ{A7e5; z!4yozreT+dO#^M8v!+1==3@aCVi6W22}|$=zQkAf8sA_kzQr;u$9MQ1E3gtj;76>& zYOKLeSc`R7j}6#}pYaPeVKcViS8T;L{D$q=ft}ce-S{1Q@CWu{ANJ!<9Kb;w!eJc2 zQ5?f@oWMz(!fBkrS)9XpT);*Ag-f`ME0Bs z2tg>GL3TWg95BI*=a3U&u;6*TfLzFpJP5~&cnL2fFCq|$SMVxcLlj=e8_0)f%J0Y73DR$~o*!dk4udThW({ET0)37fG6zhW!4;Wuo@4(!A(?8fid zgFmnr`>-E>;s6ff5Dw!Aj^Y@O;{;CP6i(v|&f*--;{qW9T!DmST*Wn9#|_-X zE!@T({EfS~hx>SdhjOwYKPoLJ3y%NWlZE+dk1X(~jk87;JcNp@ZUt9XBC;aI|5T)K zCJVt?$%4mZAu9r8MS%a!Tp`$zD>(lD*jyo7R<4j0DTWaZhGPUqViZQ>eT+d?uJAwC zt8nHDAz8UXRs_h30RIoi75HT=`M*bw|MTGEq_<5=#p7UgR$rhK(gIr;Z`PUx9z&--SRpZ5(lm9(Al?0R2o+Qb*b$CMiz z`)bmr=>rRFtMbK9#V_qB7dj$G&)T)d7QNGA^2hZDe!FY>h;YBx&dr&{^&7ZKF_pz+!3oj zd4yhbaTIgCINW~KMdxe3YS%NPTmW2FYO#p3SkzgJQHz{mX{eYqbXI%oFi{#NCJp1O zPtP~2euvJ>M03SFzUchwUkK8E^|rl=&1p>#r35i4A#9vg7bxih&ALE}MnB2ux7A3= z;3paUwi+k}_(=hNTLUQh`$_(OTm32d`7uTFdNm}|r-iJuhf(TbH1{w@ew*DoI8>cF zQod-AE*i`i4W$-Y|1vN|^O_lZAzclL929E3Wnzlvm9jE*q!4dpistoro{UZV`JnRs zTa!c7sUzh!7O9QJ+{O|`S=lIsR+KZkIr|4!@o~+z?B$KjX&fOnjxaZlh};~Y-J;8p z?#yrXVk*Hhtk8?85X*E5PccnlPH&^s+i31>j9eU~xh!|(x28R9-j%=68jevqp2vJV z4}oGnfnu{upeSn#6rmntmXGfG2z}W^wHYZ`Kocr*jeqhmpev?E1YNAX$Z9iKr62b( zp-^$PNl~vgg4S?R8ZIUc*Nu5e2y}*mU_U9?Z)-52AjnS&^4l6jy$cMVbErFPukA5Z z^)m+PjKO@y5W6S?YxnSxW^DMLNxEk;-!ny4&!(*a&8Tu+jZ#;mxvMdDzO{>yDVo>x zAD!Eelx96GEK&=LxrHT)vL?30Q9f^w&Ku0<<=Cc#1X_)@7OAbp+}1*|k5TGlH1{!5 z>}zC-=JiA#57TwtO||b6nB$1$kre&>BtO5cetemtA5%21X;-||Z*^f|M|Kl=&m}%~ zbQgW569ts;-oK{y+{?j7Y4yB`C*Chpm2FloDpf9Ou3R)l#49~0l<2W)tdD8CJ$l$O z9-XO#Eqa8f89hX+@6b^$p`(av=y01AXv0BqH<5wAFyfr+lAWoEhcgFJYYg#3X<580 zD&b&vfKmbZ@*N2aC1+|dM3jbzNkf?DeLRr*i%I?YL_bmLCnoiaS{+P;u+8MEbp#Gd zj$lH`5nw1eHbTjI;n62h@(DEi1QH*6 z8KquEb1yk9>S>gE8qGcBqH+;aOjX0uHRm25cbxLbx`vc7;mOwRc1NbO&TAIwn#Fw0 z5_uq1{OfL%x*N^i<@k5SB3-eVuUI(7%NFUf#e7*VUa~Mn^Xf75sPC3wF~{*~cSnvw z@!=_^*~~d>kj@&+XAMLL@8VFlUHMry)y_;xEB18`x4KRf_%WmBnRJx^GKchI_%qsm z3SB0-pOTPv2=Vnvk$e<)jUZ~ikgljek5`Qq&hCgN@Tf6Xn}(>p1>Kl^Go3ktBPO_X z7u>`IT~T|FcvLc1hx`6Q$|i=?$`KW!as(+oYpQ!@^sL=hc6Fwd9Ac(DP$K3(*8+7g zNgYh)4yLHjgR~osquEn?7wlN(O+8pucN0@IuWUt`q&K-vEC6SxC~Mc zmt>$PG?%TbU$dru$5B2NC7p^gpNfj4cdUeZ4$pRqqIp#UJ(s>F&?}*lB{l2IVA)Z* z>kam2G4u~^}(1weQ zr7VTzltmiPpHk6{`Q(-wJE-6}^-QJI{srxcvwz`kq`_1Fg4JCSq$@(=mB^C@esGM% zr6Xb}=a>&W&O8Ho-j2;UzZf+`eew%HvxfSV9~z~HM)N~sWSbD{1Eci7Xnr8qINN>~ z;x)57BlgR-2V3r8Vv6Qf(;v}}wM5cAOVzx3IcJr-R+t(n54QYeVT$I}R_*MsE<+Hu zAd-Qa?`g0_$>!V)ws@Cr*t3#4EbO>oBmc~%L^-}MWl(}*s!%R>e%>pPVw%nz$B=O( z4Z+Tg54&_pmUEH0<$hx#6hOgfZmw}K%F|N6YW*Eb5zeO7a>TPfafkH|U zAmn%$l2`fw{VOZyl(Xk-RFpvmXArpil~AytwW_aF)wgX`f`GTZ311?eXznHl1x@J4K$)@5?5x2N{Is=x5)t4Z z8FoqC?a&Onq_cyejYM+^H;&_qtdcE7?;^#IQhd0X{)RH@;;s*&UA2euMt4C@?%=rg zGx%{5b9UsFcH}kh$SV&;co%X?uvn^K;&x69izx}q-PckER^?K+t}7$0D`Qz#hH$Zr z^x}e>^x_2%=|#u`m-M2rcGy%m#x&j;0Msmi8lwaNkD);4<=wf1^6e|n0}GjQZ_a&9 zXRcB)8*ShRCCyBC(KZz0_M3ddnG3lP7>>JVjOI11_SL&<%F8{em>d#W>e&?I`}{bG zA%3ONjSSCjjWPH~rW@ zbwV(kM8PqP)U=4wwIH;+>smMizavhBjZeM8bj>+-JkO8AQ-@)>mvbpXx)fo)6cJe@ zSbLvM)1gu>BaToL-<4p<+)N4eO$u0Dr`i0d475bf4e~M$*+*K^*dvCO2q)9YK7MdF z5^_B4fa06Y^@eR>P4W@u{XKlmo%`OEAAdVnxp&VMRrhEDbCl5-`TAF~8!l%hb~>_{HEyyPi-#;G4_VBA`C}q#2*_B6#GBBf z-u=)%C*t|$aC!%Cp1cjS(B>#@Vx;jIZI?wJs5obk&Kb<-`MFt- zBXG(?Gn!nA5PA2j^sdgE<+-U~ZDo;KSm3sSd>ZO|zueV>9Wg=GvIB(_NK!J60jz z7FB~TNB++a%e1ygtu5x(7KUYZ=l5nR#A4o^``j$XvAIt2$c%D}!QH@2k6ro6Gvu2w z*WKh9IX&bVWn6E@6fEi*Eix7i82uRUcrck(UM;$fH)(_brTS>I1huQd(_(K0WKIas z+{{S*tdVmRM}AVo-SZ(CAI3>n8^PeYdY-g(zVx8bNZ&y9hADMjd0vsC(pFJ?vbKr4 z(n|Q0&x0oBJiYtF@_-HBajIuBT?TB}6D9lRASH_=Ssn8rS?u4!w>cdQv%Op#KL(YFvW>u|bZ0N`RN2lF>wXB`9)`-UXbr7CWRq zJTlka4Vyb@M-|`FDIOk))dZEW@o7CW;x;_+_=Xxk9v6(9Ify1w>PhC*WQ+HW z(tV?uz0y?t36%fY2SeoSKF-1v&1*btfd@xlkoWh(`ab`Mm)B`-0$C{yOKc7;+ zFU@x4o`m|4%asW2iB0+Zi}EZS*Xi*n>3Ed+cofBBQPQy}^D(*TeE9KvgmgZ_d_IDi zhohv!QRc()`_4O+>f>>Y1 zaD7s>kal>L-iqddv2+9i=i}`BF@NHR@&rhGH55=;9Z!G|3LN)-l!U%f0nJJE?g?{N zr=(|f(|Q6#sn#s3JYQr{Hc8LH-Q8bD)55#TRisUaoWm|pQp2^Mji$CMzi}fEXK8K) zXa;KCH#V8{1PEOWc|0Ypr#_Uy6#18LGTO=5oK`18$r@6Rq(qho(4MEvD$7GDN|!C& zLn&!=JT@}l(KD5Qe?a+N0wRI!d_}uEcm`Ft=+!6eHE~{!x{k*lBV*HZM>ti4_GVOt z7$QF)a>`x4aX;-S;yc83$fJ<tju zLb2BQdTG8sV*bHNRCmvJg55o0p6=h$bRQSeoU3Hb{SUK<5?iz_V_Y)-sIc=L%@rXJ zbvcq>*WZcqJm{iH!!=zlC6V%6p@<1)YKn4I6G$0WRVI0e_37k3a>D=6Bt0~lADTRm zwXk63O|4i}cazlJWbST?n(8u~EdL&k<_VaL2EYuCvP^!)`Iuk&H=W|)7tQ&S&$H5| z@at5{7fV6@Zk3@H*WWv;7bVq;GS`cuX;i;_O+YNYTh2$8l8-F3-|lt%=8XMevm9yW zBfi#9{XLf(FWRR^sMrOSWP#d|sgI2462I@~%py3f%;`%gCx_kbVwAcV&0UPK7ER60 zcu})vwWxNxA{ji*lo47@tnf5EBmZ)jCPvAvmSbdw%f`za=*|mClc{TCqT90fBQxn) ztJ_U!XVzqG%2O%-U1Kst38i%*=lI_plhJgdV%{r2v>9E|G(9`NWn(ny>@R+{-?ABM z)DL}G@YU({x<`6{o!;QD^MNk}i133y0$_j6P_s|4Q(G1Pe z0xi)Bt^g&tJ{A7e5;!4yozr!*QIzNu0uIoWWU~!+Bi5 zMT5Wamu^#>?h-#-#uZ3N##LOyb=<&B+`?_#!QZ%xd$^AWcnB*VL1*x{>i%QX9-Y5E zOifNJOy!jSw8>CgoXWaR{|(n^g>z5qI?ZaRsITSm7T!h$R753IMio>=HB?6p)I=@R z#yhBkcTpGhP#+D@5RK3n@1Y5rq8XZ_1zMsNTB8lxA`b1)9v#pTozNNa=z^~3hVJNr zp6G?%=!3rKhyECVff$6r7=ncVXpEAS7?qVuWu;PFjcHt=S*aA~CM%WtPp4AZOtS-Q ze^-GXpAX$WM~}}(Kh!@fmHL;Z{F$3d<+bV_$;tF1-6J{Yu57xVp9>{O9vYBe92ZKE zerVoMmeK|2MgA2Uz2094@DcrVz7Qb75B{Q`SWpNsh(S6dKL#QQ!N`UXgyI=w$Fs-* z6U=xHIS~d6p2rKwh1|%4aJ-0@@G|lu0+DzHui`aC;dQ)$eBduK>GGoh3Zf7SqX>#3 z2E|Yuu_%EzQ4*z48f8!xPAxOYb48w4Yz(|b3XuOXxNW@rtfN>a)3HT5b@ewBBV@$>;n1ZSJ6w@#rGcXgg zFdLs?4nD_R%)@*vz(Op-VkBV+zQC9G3SZ+JEXB82hUNGU-(v+<;s^YQRalKR_z7#V z4(qW28}T!K!6t0R7W|5>*oNP*9XqfSyRaL-V-NnoUhKnu{D}iNh(kDxBRGmr6Sr_1cknmv;vVkf0UpANM|!^nTlp6) zH!(oqiC9c97{&6Q7V-W)!H`Sr;r-&C_sqNX2!;~k*Y-sPLn$%4^HYMMoS4)3V`gmZ zpInDv=pGUN@7;a*R0)Q;5s~f>K`-CKtU8jVH818#Gn|8BNip_CQ70dN}~+Qq8!TOExe5i zsEA6aj4G&#YN(DHsEJyrjdxH7@1id1p*|X*AsV4E-a`{KMKd%<3$#Qlv_>1WMI73p zJvyKxI-xV-(FI-64c*ZLgomsy0q5X*tiVeAfFH36tFZ<@VJ+5SJvLw?e#S4@gw5E3 zU$GV2@Ef*c2XEM0!+ku!Ls%hq304ZaAi|2*#418iuq_gK{gL#n zK5JR;zlzl7N(kyzp=0XVPqNCV`B@=oPDFs~?g~K*#N4j4p5`8C*DVD7BCbmRriGwA z-c|=8=%}sf|8J-{!TR3?T`-56E@G|pEH4Cy>8Gae0}*27^zM`Q%L~D!#EEGyE(BK) z3pn2vf~(ku3(|FhoF7lmal2bq2<{-R_kMLkaPOyi>fG&NX3rCX7urS#+?;SB_$ym} z?dGk_-V0@YLhv@*I8r8TQV2dO_K;hA@4rrEFEZo*MS^Yp0{;FM{{`Fn1qDoF()j18 zbJ}hdWRv>^+2npfHXjlNUkDK42Y&>>03!ksgkbsz^@MC8^btb&@fl>tv&aDx%y-CKtU8jVH818#Gn|8BNip_ zCQ70dN}~+Qq8!TOExe5isEA6aj4G&#YN(DHsEJyrjdxH7@1id1p*|X*AsV4E-a`{K zMKd%<3$#Qlv_>1WMI73pJvyKxI-xV-(FI-64c*ZLJ<$uj(Fc9e5B)I!12G7LF$4)1 zieVUz5g3V47>)Na28kGp4=@hnF##W9B0j<-e2mHX1XC~-pJE!OV+Lko7G~o!%)#fF zi+Pxj1z3nhSd1hr!58=vU*T(fgQfTu%di~Z;d`vWO8kHyu?nlP20vjf)?qz1U?YCU zFW7|5*n(fN72EI|wqpl&Vi$JfckIC**o%GGk3Vq$2XP38aRf(k499T-Cvgg=aRz5` z4(D+J7x5P^;WDm3LNc!68m{98ZsHbh;|~7DUEITcJitR(LBBv3q89~S2mvcY#P_tH z6GEQV_teh#Z!hNzDI>>9&7lh+Z`qOo$1EYFlJ-aKlK-t`A*5GCHv9J}g!HxD7f}yp z;NuQP2pK8Hc)l2fknxXSWXj_fad-Cquf21Di>k^W_?;OB1_UA3J8=t*+8{L__(D*8 zK!R?Wsil1r}mz@oS>-(-+ zO__e*Z2{rioP|GJaRK4wa+=?2+aa@U-?xI_^|*H7a&*9Fa)rgq(LpzLp@RnrT;f3j zu`nJaaM|*rqm%)~Z-3g5IN|yk|5vy~qbB>?=|7)8D>Jt2c9!tZ2;rY%VfbeR{ux^D zc17V=V1K^<{C=(Oo4@!r2C9La#c1PSp2h}zG9p??<#(vZu9gpT71;Erxt2QQC%qcj zg9?k96%`fyQiMgZb^qL22oI#zgHQ;A`VbBga1}IwUx5m4xEdlsgKOYgh=NAY7^2~N zxB-3*H$n`=!cEWwn!?S{99lq2Xa#Z58rncxXb17o0TLh)IzlH%g3izdxp-T9mZr{N1Y17AWVdfe>YP3K_9X-ThG((9J=wk5r1Ngr9#DNFiBN(CjMwDl)nPPm_bLfvci2h)uHpiu7*eqp_V zv#`jn%#M{0w(Fl|wqte<2mmJpLJ+th7(|~C3Sn@${-7cIRJa-fTSE8GUX zpf}tBeV{M&gFE3a=nn%R9qxgF@LRYSeh2r#{qO)}z=JRtGGPb|g<&up9)jOP7G%Rn z7zKZT(J%(a!ox5Q#=|2p0saX82aiDxHo|LA3jcsj@K4wb|AN=y4cH3X;7!;LZ@~`O3A^BJ5PiaK*aLgv zUDyZv;Q$3U0QZb=<1DaDd*x1_r*X^<D&D zbruvAEsgkzuMvXvE0hqWM5{iwcVIFB;DkU30v7~B2-Jg6s1M-~0arl-_!Tq+72I$& zXmAZ&3sKMr8pCxE4cEgBa3jP(EZhW5peZzio1rU4=@_Wz*u+~#=&@a1RjM6@PF_a){pH0I$MED20E(Cio|8hJV59 z@CIywt?(vnhqqt{?1WwLHoOD>hTX6i-i3Xz9}d7lcn`{;9R36E!y)(p4#S6V1U`bJ z@G%^#OB_EihYrm^lM9M8Js3u zWw}27PYmvA9M#=Qafc-hloFPiKmnP-^@VL32ad81{G*k8vL*f5k``Lhiu#kP@Gd6MLP=` zML3^?X)qmTz*CJPI?hy-HY@_#w2oh*i*o()i?+_0iSs&!E6&AQW9K3*+_?}Izy z-DKO$&L=yM7nw~bGf&-5l-OcQJlTwJ#E)={Bk_X70+G`^a?*341W}{uo#%*5o@x}W zm+D=K;ut5{Ip-x7awsj}5K-<^ol81ywXq`0>M~P_iA(G~t~`(}L)bk>d%`)pb6QSW zV9t%soNUL0Vq5-Wju=f_l~NWs3C&Ybezqg#x<=hPM%yMl=5Q`f{9Q~7^knF5+z1CM zYe9h==Pc0na~$f57H5evI;PvvQOvd4C{{L5Acu>lEu?8TqaZ_3=h#z_cr{7QV=XQ9 zuvfKnrnGYkHI)Uj+#q$7`h~8e`DBEq>up*RC5cu}rLpCKrfpG{p{Q%5qDraA zbBVo0vqD+8$~GXabG1~EINoqf>*k-8k_rb(5%*$4wG8fXhrG)CgM;t~y7%P57R3if&3~r7bTM z<|-{d4ZyIukM6w5E}6KUTXJ75vi$=Ms)S^ zJ=)vRbi5K_n1#gx_LR7xf7_BOEc2 zTATNYPc<4(X5Ds;0$SgjdJt9j#4=?82$;t-UBtbpd@+%>$Xuk2%P9^NpgX!%0)W ztg)vwJ36Xb(0Ql`a7Jfqb(rMg)z#?{1Se|45epOl=;gXbR&rqQV|1D0Vw{W)gi;OX z%_$GeiE-+t-$izrVxUp=yo>)X&%4b2vTD7p3+_75>MA?nOtb)pv3kiRY)akB@T7;G z{3_c!#pp|@-W}2>RM-(+VsyV3WQXW{TH}{hMB*kpC z)7JVxB7dA zU3m=UqigQbE!XvSYz%^-X;^!xt!ou*wi#pE~x&^Izw=nxg zr#9-ebVVP%sP?hQ@RH%H*|)q|riq^XA+o8lCru3EaNVl)Mi3@Qf0*=!Cw?@T-u>AC z2DO8@T^ODR-yYK=~74CeFA6H}Yx*8S3})CyIX_%4bQ zHLAsN@!iJHyR6Hu)c%)VcHL81HNr7ugd??&He48+Y01{L7n&OtkzjQB^Q|F$_3QQ@ zeHzr5;w%dJ(7VfToha(1MHnM@YtVj?-+RXSoEx4_XXLJ~s@uptu|q)X*6~}#(BSfY zBxiz*E$gm6mNl=Dyle?msarS6snLhG7xj@wRH$1YdBvmKnmb7RXOeLxF8#6_Dlq)g zyu9RkI-F51Bdive_@y~m#gMJ;Q(TRgqMMp8GA}>SHLeQ0J&@W_pHjW&Tclr<8&e=u z$D9HoqU2CsU1mY9;F?U1Bx?7=6Vlr+_QNVU0V15xh%@vbZ*(Dimj%XLYfgmFRK|OK zjPc$Z3>t;%MHC~bUuKMFDV~Fo2b9?8SXnucyJrV$s^7k6_pExi=mRz~6g&QDY? zH9%E(f-SVUok@;etL1VHXK`b!TExeV&faM}>K38SR@HqW+^)(kUY)%*dqMWKWyJ@J zcg8QsF3Tz#QoOS`yI5{6%UZzZ;_P^_xh!7aJg6+5)I-E!Ww|@Vac1(iGM6DTmo55c zd0bhR%X4k4>pBnC8 zyNV4B2j+fLy0!G=Hx*D?0((s1|=+OsIfRq zq!xMiHU+Zm*D*eP7yT-Q4wo&rsM4lXb*uKT-2QZ~O$pJG?Yy*} zE=CdZQ@pws@&_Els>Povh@!JpbC~5uzmT zmv%N{ifvHFkbk|@LjO^wj0Xn|{;BZSi1aUHs8+Cw@Ee+sUOi;Md8|T_;{c z6#Xaf>vBqetTbkr_1DB>R(_p$bYa?^UnhQ@n6}{S-ESv;JMm>7=03lj{4d)H^DUJT z5fSyP{WYzOjyPrGzy9gEE){bsh+nx8vpYKccBIzR!GrtWcu(UwJ1BtU$Lhg^8e91=^l(v-Xi@vmKF(;^lARYh~>MK_J*izCeDTB<^2TEazYil1hi zX1lNH>}fXTOo&|Er!3{OR2lj_m^Q4Q{viZ;%k)ZQD<}C>gpt$tl4<&oH1UdTTG=3t z?`*r6NadIfjo7tF`>xeuv4%s?iiTFsx7E@P*op$GwDWDf&ZIHB=u$$&%%GTC6pv<0 zp}sDNxy4{zbX^c1&fw*`5M~P8NEWZlok{=PVjX8r%~0;2Z}o(9zfL^5xVX#r+lfav zemn8#!fz*+>cX!RzfMdWsLQ*5Zn4Ijy=Iv8&vJU)$*&WSE=%^}U(-wTa`|ZSU zC%){%+~>EGOYMaDmP+Oot4%G|XKqoSRjm8;qBRxdJz3`XzH^H%eNwTW{#M_dTMW@B z6~pv5zFqs5KDSt)&n*_%l**se++ugHxyA1M7KHcwqOZBdvvO{6MZkIOEOU#!+%9!j zv|CZ!`=ZqY(QckKbgxX7QiS53{GgN~nWa4+;a)SvrcSeqw55E*Z_!omJiFpPw!o%V zMY!_<6!*~;Hgy{!znHS}HJiGT1myPVE{;MBQQVd9+tlTVp^AIq=QedAVtvKEKHsjc zM5Mf<&)C&Xhz%6?@)zvtM8t-QyQtW%79hG6_qwfibtYn@;(q;rT|Go;*C_5~pV-w( z#3;o*|D0Vtg4kHW<(A~x7w6em<=dC#**D1LI=N&E+t=jVm*?3joxR2R_NBB2byO8N z)LjikBX=xvsM{LIiqBdlH;T#}-t16!yJP`rd5z^C)AC8~b#FP;WpU!T1K&8*HF2_) z1uFv7LX;{L*vq|UM}Yc{LnO(6M@m9HyZ7iKtzWud7e(sk?$hrDsM}=f4HZ&CL5ddT z&OaQWCXk6HNOEsrbxEYi^u2Qd>h?%k?y_f`>MT{1tHqZ`mq#CE!ad8qzR;;|3l~ZA z);rZz;qpwa+Uiux5jiK*_B+*y=nn1A{Mf1PU@LkquXL(217(AU?C4t2buWJvU1yVH zlzZ8?PW6be9?gArsA%Gp#ewR3(wq*LN=YBgd#lm)5l5y|hs@gRBTv8eUm2B{UKrS`LDxYWfYz@pYY z=Tc7~3Z1TYsi*Ze&D!jeX*r3fce~U=5@1pDkGRwWB%r0I&$!enY{jC^O$t`$B4SZb z&k0t`5v4_~l$L|RSZ0F>>E^@|8AHT+pO+vkO-tKwuU&%GIq9&h^l2FgS&vlX3FaJ^(b&IMz8)8^g z>3Jy`7BxAqp0`DnvZ`BD#at-~?N+U-r!F=u>ab-|3%Ar$w@ZsU`ffdSlWtMVKdPrz z=oU5c%X;b*UHj7}g~|lzdB?0!b+2JjySi3(eW&XIEb3`#Q3n=;nih3xsG@%goMD8?Yh99%&ywSW!a}0>)-OGB!r(6(#Ozhz&ubG9JwunZfS*YN%?DbdDN5 zd{Aag2%qwbY<{m;ij*P`XV%C;8Cfw6drfN))m+>Y;iiblad5`i%$U$#Sz?38(K+Jb ztgJCc8SPom&Kx{EL*)zK<=zJ$9h*7EsLZI0vBNwG1IA_y9i!USz+Oq@)jl9HAZz%T zv8vsuXl7PscIF7P^6tEDT61khtP(|wV~8iBjjElyLGELZ+ui%}ZLzMTNY{lRQBg&S zv>r2eD@|(Hifi(HV2bKel9~o`2Tt5(bFX+mOzr31xG_}SZBs-^p;~(-FhbL{s25hT z|2p@QDfU<*eQZU(Eh_%Bt;zq+fG*Ok%1=L?PUIDz92BQh{52Mf2-cF^OP`6>(D`ZG z48AMHU4_zjINUQzL)F*Q46V!aYg(#tl0UGyi@U|r_awzDO~lPl@omBvyw0OL?*Wmm zek!+r7^aSL@17W@=If`jzHT(Sx$9Z3{HM)bm1m2kUrLIUcZ-8A=!pPvm}00 z%wsVdj)bXc?!A-3RBpL?*{1G09`oE(KbEcy`!sJNdn93>mu@vnt7e~hABCy+x(`hb zGwsu^H%hvsUf7vDn|o$ZEIngVq`cRxFDb-p&CEQ&yD8;l9>)MCOda5T7Ig2{Gf!|Q z#L~AW#p5l8|A_Oc2(=3bqJKOZrZ#p@nHr`x^U9=&9uk}~6zEy)eHY!Q#?mz>#aG%d zJ)daVy*QnW*{c*ZBI_cu9^hU>*5fYCIzU7gJ?F)H9PV{@#nO|@oV{;QM|$7)Hd_!% zNn7^Vy^G5GI80q;a~BqbsS_PWVKNXT$dDz&RFDy#DD?cD?xLIA3$Kr*ua7-`-Wi2{ znsBpP+_h$y;_(9A{Lc=(jd@3id5k7R z)HCi{V^v1n6IONURM{Jf(x|uWgs{)K2Q2%%Y}8})_Fbc1-boSlK0R1V)MNbT zBkHXplg92%@6{Uh4iWV#QezoDCdHq*Ow?N@quv?2GEGFiU1UAL{V7@dj(Qu3dK3G{ zGTM|m*D30i>ru}vYS}3n^$wNQE$W>$qaNeY*d+@>Yl(WNWYnv6Wv#fkeJ-NjkqR00 zHtA81h~{1=;$Bo_XmY=PccgdJhqjT20$Yh!yX0Gvlau>`c-@wqJU%&PXmVO|^3dd4 zhRSUzDapyV3yFlk5$~*<@2ve^Y`TZ!$!(HT$zeRZQ^t!N`-t^qvEE(gbhk*U7d5EW zxK^!LN$H*@$n~E6h7D{hi%-6*Plr~m;vTxWRjUN{rl!f1R938r zh5i!xrHmTfW58Is-t#WfCyuwu>nGF8g;fljk-rAyK4n z)xDPX$@)D!IQ@|^gGAmb6F4T}9&swg+AVseYOLQt>Nijp-#y((PtETV$8%`vZMr^1 zrr!FL<&Al;P48h3CMVN`p~61&rU;YhX&v8F9xtrv(&NVu%FcR7FMDWmcex+?l1XKG zQXJifs`%2E*w-^qFYhsDUJ+(ikSkkJtS{9w*dgyp+i3Q5mEE@NnkX7K28OYLPK| n^vLni<1$B&89s7Ebeq;K<66dPE4Y&M@vqF}F}ps={aXJ&eYOpZ literal 0 HcmV?d00001 diff --git a/matrix-parquet/src/test/resources/optimized/cdc-1-index-0-alternate.parquet b/matrix-parquet/src/test/resources/optimized/cdc-1-index-0-alternate.parquet new file mode 100644 index 0000000000000000000000000000000000000000..8d6380d0d417181fb3402919a8fea55b9c45b4fe GIT binary patch literal 126744 zcmeF)2YeId|G@FP_AX7*Ng<^L3S|@ulv1D+%3fvfy*Fi+4Khm0kWsccn1ZrFREC10 z43QyQ5!nJFTL>GJ5&oacC24@Rl=V~p908-fH2a7U|T&S1r=3MFzDnsfC|fWK;`(wFpp) zOlpx?EwZRZpju>Ai)?BUq!!uLB8OV!REu0{5v&%u)gq5tgs4ScwaBLy`PCv+Eefbb zm|7H6i*U6lq!xwMqKH~Vs6|n=h*XPWYEfJ*qST^wP>Uk zjn$%wS~OLQW@^z~En28WOSNdF7OmByjasx-i*{&r)j4-%K!bnfOFbZ#DG{#^o#vu{oF#!`X36n7eQ}GU_;a$9k_wfOy zV+Lko7G`4(KEzzi!$LAwh-?Zl3zA74(9^(?>9J(;Xi0gcrQwgLLpkdSrkJe#i)a z1RxVKBMSnN71zL)i*ZQAcuc@VOu}SL!Bo71X?Pd!;eC97>6n3;n1$JxgAXwm^Y9Tq#(XTm zLM*~!EWuJN!*YCr75EgN;d89SDy+sDe1R{q7VGd8zQ%fdgKx0`8?gzS@g26{du+uv zY{w4#fSuTd-PnV@*oXc25kKJoe#Sxkf4({R}{={Fnj|X^&N1pmp7F&8|Pt(ZTED@%QnL>EXoyB= zj3#J`W@wHUXo*&6jW%eDc4&_d=!i~;L1%P9S9C*n^gt|nq8ECj5Bj1X`eOhF;w=oq zU<|=f#96fQ49u#aM!+Scc{J1S{|N!S~pTZP<<-_yIey3%jugd$AAu@gsi10sM@E_yvb>7)Njv$8a1c@GDN@ z6i(weoWbuni*q=S3%H0&xQst=1y^ClHC)FH+{7*1#vR6PCTNOgXpRXpau)h)#$>XLLbV zbVGOaKrDKq7kZ-)`l28DV*m!?Eeyh748c&uVHo0(fZ-T{kr;)yF&bkq7UPhJ@tA;# zn1sogf~j~1)9^0d!~6IE(=h`xF$=RX2OnZC=HVlJjQLoAg~Yr?ws2+-VCsI0l|fJJ zw^%Hzk!a*o{5di+$LSAMq0o;Ab4fFF1t5ID(@% zhT}MaUvUzra2mhi41UL1oWprsz(ribW&D9FxC%3_;W}>MCT`(2?%*!&;ZOX9`*?td zcmxX`d+H@YD!-lKMdYRnTp>Y*4sOuH03+Pt0Z(|r8$L(}U!+F{nBa$u@J9eLAv3Zd z5LuB8LCB6A$cbDCMsDOm2=XEy@*@-l5Qc&XMCfiG(&T=KufejYqUXIv_pGz zKu2^!3_7C=x}qDpqX%Nq6TQ$Ieb5*E&>sUZ5N}}+24e_@A`Zh4j|2?I2#mxiyp7Qq zgRvNgM2yD-OvEHi#uQA&JD7%d@gCmC2bhi-n2A}KjXC%bb1@Ge;bY9l0xZNLEXEQn z#WF0%Cs=_`@fkkHO02?atic!f5^J#zU*T)4$2a&E8?X_Zuo>TB3%kLNqF)3aX+Ss-p&Kq84hS4(g&F>Z1V~ zq7fRS37VoAnxh3;q7_=B4cej|+M@$Hq7!1!8C}p7-OwF95R0DZh2H3czUYVk7=VFz z3xhBiLogI^7>0NxU^qr#Bu3$FjK&y@#W*BlJSJcwCSfwBU@G3hG`x%V@IF4kbj-j^ z%))HU!H1ZOdH4t)V?GvOp^{hFk^_6{pyU-9Oe>6`rp+#nZRyH7M_ysOyOLKtyHv?5 zjMi=G%`V~+S=p9e%Ihq{a(sdn_!OVvbF9QFti~FAfiJNZ>+lu6#(I2%Z?ORzu?d^; z9k$?mY{fQg#}52}o!Eul*n_>;hyC~wKj8p=#zFjoLpY2hIErI9juZG5Cvgg=@f*(I zcbvsJoW})R#3fwDAGm_6Fyk7o;|6Zx7H;DX?&2Q)#9z3N2Y84_u;8&Wrs$$~NLp6G z+Ngkvs6?xdwl!FL6Qw23EoQo%In;DBX%Eq3H+!qDV!OL_mhYLR+-rYkYpcGg6S*s` zx+OY~jxy98HpGS|!sLa!peili(`| zi8)G*BOrEY7xVqH0)+~+Yu}-xMH0-C5HL|FY?&x<37jYtanvFlwW5w%q@z~MQ7i7K zMNu;^TxMh;SC5-DKvU!NSY)I)$mZ7qFsJv`w;3P|&Y>tzc#Zen3n}^BE zhEa=`&GGWGcxnl}2pK1; zt2)ZMs+HCKck#EsmGHm)tk%><;O>bEZa6~X!x#bpKK`p z$KI0qQ5pKxtv{jYy;2k3Z9KHkq1kWwYz;d9(L%rO=E45w7T!IxqL{#1KRsSFJ=BMq z#)!5Jo=Cm6+LLQ#tnii5LCTRg8AR56fK;hWA~)0qT10IY>R=#@Gfh%Ebl60JChjkC zDeeEtBZPHa_Z!iau4~@(uZc}fXLX^bhW4%| zXqve4yjO%Bb z=v<_ zRdhPDPPbD>&7KA_ZVZ;qgXLv|^L^uK4`bxINgK^CnH>|?Q_t|8`fE?pLHl}g9s$b4 z`Q&zWw+u9z2O9kbDj&kW?M$`5(cItY*I!whYBu@YznXctEv_r;AwQ743*K7#8b>lR<2R4(OS=M5}X1?z2 z%QM6h9-p|(<>onq`JBP;oFQK>U(0!e`MkmJyn&T+E(4b^u2Trxi%%=jRN0X;hG?qo z$a&6gLVEPd=q$>P9A8rJK&euEQn!BN`*9rmci+T*9HF=)@;+~jc;&q}PEr1)d&Fuw zf-QtUyDKO1f4&n(*==;$z$J|9RO7E>Z_ed^{N5aOmLj&E6b)K%8jbokIY69nm|bK6`*JX$#i2 zWO1LCEIxnQi8gLe79UvmSu#D~6joB%pYuktMW?}8DqPI6MC202wIfAzYJ`sF_S7a? za{F^$f3}!3YvGhP+8Q8vV{)^Z&t!|FNcInwQnP-QkI@|wpnRQ2wHs$x+9XYA*y?GrwaN5W zrFvyb!7;7NwJEPh2M&H6=eYEypNlpiP8YiIe%B((NZ=(Q=olskoE zE9f6_qP{ler%m~3XZdMo`K9e(Gi}OGoAOKbyo0CCO3(Wd-fd2=qcKbf}v ztf%PwW4+q1dM5pkOs_EC*UkE$JWQ@^n&r!Xyl%9&M#0!5cnE2VbN|2AJT7jPOSQG9fdvAP`xR z4ME6`9LR}W2u5z?K?w39AMzs<1rUaU2uC3lMiE4yC?Zh|#Sw)PD2Y-ijWQ^Uawv}q zsEA64MrBk%Ra8TD)Id$tLT%JRUDQK;G(bZ%LSr;RQ#3r+Fc5EH5C&rih9VBb5RU{5#|VtXD7=l)7=y7G zheV9W1Wd#vOvV&U#XFdWckv$H#|M~>8JLM#n2kC35OXmPAK_!n#{w+GA}q!dEX6V` z$0t~UPw^Q($4aciYOKK*_!4We4qxGGtj9O_78|e;o3I(*VGF*;R&2v|?7$D$iCx%@ zJ=lwV*pDCa6As{K9KTh)1yCvBAwE{B0Mr;KmTea5W=M!Y+XwpdlKeF`A$$nxQ#bpe0(NHQJyp z+Mzu^j zMmPTsTkt)$VjH$&2Y$d#?80vB!Cvgce*B1^Z~#B!Ab!Ci9L5nG#W5Vm3H*waIEB;r z4QKE>&f*--;{qeBuq*h)vjxe@Al-uJ$^cYc7jA?6X^eb$1}A>3qge(_PuN9{qJh z1VJ34mi!-~Lda7~`17T7S(LzWR~Cf;(cmGt$mzta?2nLS4?&i7f(~xb!vG`P;Q>#0 z!5cnE2VbN|2AJT7jPOSQG9fdvAP`xR4ME6`9LR}W2u5z?K?w39AMzs<1;GDkD-=XH z3ZXEHAOb}ZiDD>@D3m}+ltO8gL0ObTc~n3}R6;Z=qYA2`8mglPYN8fuqYmn#9_ph3 z8ln*zqY0X#8JeR7TA~$NqYc`k9onM3CO72VJsJrIkY=!M?sgTCm8{uqFP zcngCt7(*}=aTtbpBw#p3U?fK2ZH&ekjKw%4Vmu~bA|_!nreG@G!8E*!_wYVGz;w*O zOw7V;%)y74i+T76A7efiU?CP^F_vH{mSH(Q!3une&+s``Vii_n4ZgsaSc`S|3SVPA zzQMQHfQ{IM&G-&m@IAI-8@6Kye!x!b!fx!rUhKnu{D_}$06*g(e!(Fe#t|IFF&xJU z{ECw}h12*AXYf1D;vCN70xsebF5?ee!Bv=X4cBo4H*pKMaR+yC4}an>+{Xhv#3NYn zSk#dx%aS09VLH!nKwyWDK>3ilikx^egrOk9Q3!=m z1Q95TNEAbHM4<#qq7+J_49cP$%A*1*q7tG}8C6gf)leNZP!qLK8+A|@^-v!T&=8H# z7){U=&Cnbz&=RfC8g0-P?a&?_&=H*wgU;xJuIPsD=z&=DL@)G4AM`~(^v3`U#9J7I z!5AXz7>!(BAIW8k9G~{5lU!!YRn@yoE^}pHC5uS=qhIG6f0^f!Ty_WRlYhIC%K67Xt6RvJze&+>gsY|XI5E#MdWa$N!S~pTZP<<-_yIey3%jugd$AAu@gsi10WgI8>o!9qC4rDe7ndX#(NmB{ROgl) zAW1Rk^c82sfWD&&AqH7y)92o{plrX zk{p@(U0!6xNYcmhAob}aX`yxC{~K28i@am!`%2PJ)(uA9vLqdR`rS*?Q90|&ttv^n zC^vt(hdRH%6zj4$MG|CQ;0baz=i8EO5Q{l4NUko)zSidO#Pe(;n)@peR}QdVXCm$% z3o*^e9BMkFdtw_{w_A<7a%OwnO20o*+ZUf8O2FTyK0ivZ4gf=aOcydps zPDvqji@HfVC56y^9H$p_w%H6hSkk#MvGE!+2u3VrGvRL_=}O3(lU`cVm9cJ?r1DNA zT@~xlnCe~XKJPAR`K*$zz1$<|y(RY$N!Q)}N|LUpHBzPWzLI-}l$J@lk=6+Qnw~DD zmq~58q+1|wS3kd`Tl_jdTxzfSyuU^0b}G?Nx669Ho49AmV%nTO)O1mLA_lrWR%4)U zk39w^E#aY?ysRTOj{0 z_F-x%#H*(e40LWuB)1=}8yQZEls`*uhvZz&Q=v(ZaDMdPQIBwYq>P9hfUR6xkDx1O zTFBvr!}a!YQ7<%JyZ>?qrXL(N#zsdyX(`VzHmJDu%nrS1e~?8l{Ch7a^#W^QvbIJX zU!k@}o^HJ&+}6g$m2owd{K;dotJPjjs9R4Q|=ZS%7treiLl zrY{^@^2zke6kqsofc1sHV2l0*UO1Ka^ydDQP8p8qh;x*o$;Uo4N$#}-!QXn#hjhAS z9g0G%Ly?cD+{q&Si!Wx$HVplTKfRYpIYTGN_RU8Q4^rZl_2@{dEIm=j^e_YW@UY)w zqWHu)hvq!t2bG5`U7m0G^6|bd76YdrDHnqwR9?E=MNk@xdceF3hg7;kf(#wppoalQ zc)$~0@P-f4!58U~0VZUGKLU^mnUMv7$ck(TLJs6aE(9Yt@*o6xkq^qr90d@Df(S<; z6h;w5peU3RIf^3+B~TJ69pzq;zg0psDx(Ujq8h5hega2r>UB^T^-v!T&=8GaKY61W z_2y`SmS~06XanaHH#%~=6JpRAUC|N{0HpBF6_o0 z?8QFp$B#IGpK%bs;1H5KTK!j+pTsGg#&0---*Fb_a2^+N$$FIfqCKe|_Iryv-o_o= z#XbCqzi=N9@DLU}hG1~p(nc5Al=&}1Wn0b9QOoYAg*s}*9JLCLS}jMdsiW4xQS0fb z4R+MtcGRXgYO@@*MUL7^N9`L&ZM&oPlcRRRQM=%%-E`C*S!?6hd5mwP``fQ8v{AAP z2G|v*R0r6q$^a{mU%q_4kBfbrRmWJffx*(l$QbK%PLeC%rUV%}xIqsCjBtlC%zD8a zK1c^&q(=sr;D?L|Kqh2H76c+IvLOiBp$xOR5RBZ&gAn9JKIBIzlwr0Y!cho?Q3Mev zibxcLGR&4hNt8lqltEdPLwQs{MMR@As-P;Wp*m`yCTgKJ>Y^U%qX8PC5gMZjnxYw6 zpe0(NHQJyp+MzuKtp2!k;MaTtbpBw#p3 zU?fK2ZH&fPj6)*EV*(~(5+-8`reYf2#d~-kA7DCWU?yf^Ha^5$%)>|c81u0J3$X}` zu@uX&9G_qXKE-GF94oO3Yw!iW#9FMwSNIz1@eRJkMr^`ne1|Rg9$T>u+pz;Xu?xGg z2Yay(`|%@w!T}t_FF1t5ID(@%hT}MaUvUbj@f*(IcbvsJoW})ROv?~^lRMtR9o)q| z{E5GC9}n;lk6?jdaNFCI(_6i4!>gxbSn_w&ayn{ZhRRA;S*#AqQHHX%M^$p%qmHB2 z+)?YqqiC46NA-3*YN+ELV;r@29JM))+7d@?wWGGdQTxGB``J-D>8M>YROVdv_39hG z?Re?O);ql3(N>t=b$id39mVzPxh?B`P5X>Y*1Fg;+n^BFl}RUjerTv#SQ&Shli6l) zjJqQZf=9k5$6FV^T35)>!3}yCV1zq7;0Z7IART;>9vNVQA2Pxp0mzIj2t-z7LlCkf z2XZ18aw88ykQe!oAE79KFwl^mJKjcdpAsmA(kO$nD2MW>fQqPuXjDN}R6}*tKuy#_ zZPYsUZ5N}}+ z24e_@A`bCLz;KMfNQ}bU7>zL)i$sja1Wd#vOvV&U#XFdW_wYVGz;w*OOw7V;%)y74 zhmY_v=3@aCVi6W&36^3xKEVoniq8ygxjt7<(xSQ2c+cc8E%ymE&9$GT6He2w+^2H#=>Hexfr!xnsx zt=NX`*nuCg6T7hod$AAu@gwZR_RrK0;t&qw2#(?yj^hM=#YvpPZ#aYBaTe!r9v5&C zm#jl~aE~;V6W{D1rzSMI?$L3MEhyrBE7WP!{D- z9u-gt(Ws0nsETT+jvA65P}q7E91;+2t0pZ>rdC9r8nv{9w;<0A$SQvr z;4RO=Y|O$;%n0xJL%^0Gn6yzIzzVu>a${GHdZtP|IA{|}>jl-Mrv zWLbmr1$YPNMF>xpN8G|6CzO7x3{N(RCrf(o=5=E5I-bDxG(BA-d%6zf&a1ChU)18!Yp5@=QM6c9BoOLzLH@QhHs99U?cZuT^HM z5F8#=d*BHjOJex*^nkJQ2c;{=AJ@g_lH+4thpZJx4s#6-2v}L+xNbCSp64GK>l&Ob zuw02CamX-Nd2Upz-~y~C+j<+=)0I_LfEVO(^7w$mJdV{Bq)NQ5luhZF{U^Emd)`M938)WaWb%*Kxhxx{ji6sq1JKPsEzG zerXav!Ucy7U zC%0LDRJb|hgwCvfGc%&r1#d51#W$aU2AIf4w7wv#Reex}6Rfm=m6j{IjaPK?yr|Z} zVZ1k**=U;A_OHZ%QoNG#DHHjyCv^6<#cSDCR3__++Q=K37gbw%b06~gLRnF^l~uc! zYe3fcb)t{8GZEa!gqj|@g__=WwP&fhQthRq{OU$3y>$NaC-c^x24a{YSEc2GM^b++ ztaR8wA0dKXC=^w$B9*IR%2jdYDk{=PkP0iER4Bw1D@!U|q(qQXNwzQ2`2_c(KF{xd zoDMav^mJtf+k3w9eWu!Z)`_37)?TLP@EwM!-)@>Z&uHcAr1g(x)KI=wrPoalsG)Qt zd7RoR>~Gr9p;#emBG|bTldl`uJGKY}&~DCZ@A5FslEQzg%}3gzHoLmbw{)s;*02lG;jPYZ#!d zDD8mno~t6tm9^<4(`q=+jizAwC~Zw4Di009sWB+yF>&J_&F=Mea+*Q6Cpx*(x1GDU z#dS-t92^*My+TiAKvagt_;^>PmpeBHC739~nj@IlI()!k9!(Q2=NQ=-S>s)UGX<1m zBnZx8AJe1nIyyezs4`3?2HaDI4C0Vun0hw!tWyRHLQizP0PpD2I7R3=>B3u$=P4$X zUiWMVVz9Kny3@e8F<4dxbRwBD=qMjWdD>^=P*fA;9X%0;9Kmpb-m*ZSb%EXygQ6Qf z8-tqNW$niFbd9bX&{c_X^dN7g2U)C9h*i|vHgZ?kE{+}>5WXcBYn~W1l^IhpAlw#+ zRBe?F9scD`Ca)(btKqdB~iop~_mbMJC%DvxZ-JqPpUq2~759nATGn z$4^u=t#%JJ9kd@J7a5uAm~PG<(a_pQnO?J|U!-2?4CP#e7f8LFvaUg?W?hW_y;B_D z)l}@M(LcCgz)GdN6KtQGcJZZIS5z5mKn)Ivs=|_u>qKibQCcva&GMBvgh<*|$q_%S!{cx5=9|#PJ{+WUAAb!CiYoRSKA&A$>eX!Qqx2# z>?;T!p~`Qp?ptNOUQMRF%g?k&A8NW8Zt_uo$M1k|h|9j!{x17gamPjOQ>SpkNJ5X} zF~=j1$E=NA6FYlv4sjixNZ70)oSldwiCCK@tad$4iG z#sp8J&pK*9i|KiiYgZra<1xHO$-wH;gMER)(#AEOVR{yb0j3y!ILDVcqLxB!_;MeqfVw5aV1Q)7$Js8=uqOj6U;pYDQD^GqNhp$l6$FLrP;>%NMD%p(8Y=y)-8p zQ_SqWVvtnixFJYbD~3sXd(e<(%f+OG%(Mf(&EiszHGCh+SDDfNRVpp416Ri*#a zVoh!-y%KO)cL zr{1!&e&%_$;ujh!bE`5xPFGoJO8UszCFOV;u)m%LEN>Me#rTGqi_wV^_?BN|Q}DU= zAYtPUPoqEIDNSKxKny<;;cF-_IJE+?m?2nc15(xpffMw#A8a)AvNZwy)r2czde2sl zCNP1=1q(w{Y66kUcPaDU?PTltnp|M+HYy&_p*|X*AsV4EnxH9~p*dQhC0e01+Mq4kp*=dFBRU}l zozVqd(GA_v1F`6dUg(WJ=!<^nj{z8nw=f8UF$6;qhhd0E0)}G*Mq(7+#%PSeSd2p= z#$y5|ViG1}3Z~*6OvAf)5AWjxOvenOCl9DInmn1_$>G3H|d7Ge<=V+odG8J6P{ ztiY%E44-2qR$(>P;0t_-wOEI*@HN)s8+?lm*oaNojPI}o-(xGbVLNu<2kgWy?8YAK z#XjuEkN61(@G}nL7aYQ29Klf>!*QIzuQ-WQIE~+M2EXGh&fz>R;36*JGXB67T!k6e za2+>r6Sr_1cW@W?@F)JleLTQJJo40+Vz;{WpFKq&tJqlBv^{_yHMJij;jf>j>~!ZJ zS%p!pDOts{OZnTp0BcsUnA5vI=)Ct4MVOkgB>@E2pIugrZc^ zic;^`qK`e^B)j1|%~K6b{`z-qcejS9XO?oWqzzy0%7(9&s{AWbm84St$60Oi0ZO|6&&>kJo z5uFf&&gg=!=!Wj-fmrlJFZ4zq^hH1P#{dk(TNs4F7=odQ!!X1n0mGps7XRVI!ZVX~ zhlC65z!ef?=->uD3^2kS9`J-0yy1g%@I`uLfC+xc2!8}16EY(U0+AKj5QOZ=ft<*N zVB|&~gdi{SAwNP<0AVPIa1=se6hQ=vA`-<=98oBNk|>4JD1)*n2d)3V($0!f@7PiU zd*YzXm;{<`dxe?~*bh7Q*AMk_WEHl%D_O;}OS#v6FYAWy0ax)|Pc5tX*X#gE$|}6H ztl~d0s+1aH>(uscOwZpJZrbA?Y8scc%j+>G>?r^I^>MblTPM< zM=jD(E9R&bchsV&nI{VJL?QA0wo;*R%OrtI7}sXWY?czur88NsN#<)(;g6l?nxgqPE=QOlyx;LtNQQaZ+|P{fBRdZ@s-KK*=&NZ=xPa+mU_wQg2t5} z9~H7}+pMACyQ+P%q4*zr%lnMV(64Tt@kQ^oeed1ILs#veHHt^Djwd!Xoz;b!8d6pM zN`ukY>tb+^N66e06})JizETz&P_x3Zp|2h7a>ddD+O^ zn>~|yyr<6WsoUup0o0(wJ9#WHdiA`aLq{?&xia z^Ko1v<9zbX_ptOdatY)5nRlx$S#0fH2bYPa`}_>mLdPiuMlF1#5I!z949Y-<@>6Zr*&E@Axoej$c;^I&<|;C$bB+K2rY=xC`WyOwL-X}Of(ZuT|g z=owUE=qPD>h8Mb9JkQz9WbS72>t@Oq=K1t1#u_cL#xSm5>r40K*}4Zkb@_lL_Sl|# zdKOE6qq)D)ufI`C6qD=K$t|Oc+Y&`<93@e7`{yQ#wmwYfvd5X{j|WMci!XTJfkyK{ zqu)T|*H3I*rae(i3f;*KE~$m?l!BPLTul_uB$&=?iDI(Jp>s2*dEc-likq_962+{q zJ(9hVM3L<$+g^w#BN_1~Ml#1pqhp@fE=g)`$z_UqomsEjsV8hbyIu0CnPO4^`zPg$ z^~nSu8qE)leh-bgsXOJ2?Ww06r21Y{HpK3a5KWT}PFrF+EmL&vSoS&{>`}QOWlPLo z>0X{vRUh%S_pcY(CbQ>3jP_2|W>(0ZZl+ZDOClq&Vv^{T(cp`E; zMvf|L^H=7{F8|}_$qM}wCL?XTKdfPt-5&{m=k5>NWH|F=FFYCk+@8Q4`WnrB zjedQNgKKjm`HuCRrH=5e%#+yz;zWDOOaOWv(L1F+dw?!!3)Z$|QTab5v}Ex)#}zmw ziw~^3^8+O3u#(DTS@JVvub3@5HB_l^G0PH>OBmP5wLC|2OEFt~9TT%j!7in#GN-)J z)&R*H)08a|H0>WOrDpvq%Rit?*i%KNOFHF>7Ngl>^s}U#D?V?|ERFArq%jlkqOXZd z7}rnD2-Xp$!%5gN-$VH!angD;}siJb+m-T3` zH<2o;iDF@n^@>tE&+9(EE7>`LG&c8??zaBA_p^DTn!zc#Tgv+zQ_tp7|7%Lw;&Uhd z97&>fFjeY%T9o~q^f6nKXbqnviDm!JB+(Xu-c*w4v}x7PXzpk9>t`H%{DtyGN7R1e zV5(Qyt@`X9o|L+wx?G(jJFrRQ|I!ULt#|Bo@X&VR+jrbMmo)H8Z9@DgT<=+T<0}Vz zyzn%c(?L|PFGYMJTP$vg;6E^N3FF$HR543tJ8~}#DPqzbnXM18!IHLP_nx03ChfYn zCyUQE(YfSFwv8rwul*ya8B^pNJ9?@$uWHW=bJz58-mB1>$YkGp7`5=ZM^QcdiBexu zCY8MKg28;j;CI20yG(jV3izUxE$npUC7!%Y=4Ower!Lc2;t0W~vRR(I^gX5Qq|~PT zo*xQr;ZU3MOPYPqru?3NjG8v(m+ant=NSCFI>A|+@=N9D`jjHHJsR_OZOZSdP{~bg z%I|p{?@j-mj;&4krC~b4X}dJ_>a&eJjZ3sCzgLb@DRmad`G?w+AE(4RpBkr4`K5Ml zoHph6QeDz1S=6TdUhbG@r@T>{@_QTL0G`?ZgrOk9Q3!=m1Q95TNEAbHM4<#qq7+J_49cP$%A*1*q7tG} z8C6gf)leNZP!qLK8+A|@^-v!T&=8H#7){U=&Cnbz&=RfC8g0-P?a&?_&=H*wgU;xJ zuIPsD=z&=DL@)G4AM`~(^v3`U#9J7I!5D&}h{G_%BLTxP0wXaBZ(}sZU@XQV5#uod z6EO*sF$Gic4yNH$riNxP{xegS)tgKk*mt;{hJx5iEFYaI*-1+r=!nsl(NbG!0j>n`9lWwAT7> zXsyxb-$ZK_1X`^K7q~)#3?1B{hXF>o!vmi1f;W7S4!%f_3^2hD8R3rrWI|?SK_Ids z8-kD>Igk^%5RBZ&gAn9JKIBIz3Lp#x5spGAj3S6YQADB`iX#doP!gq38f8!xFw_4zu#f98zpy(=y;(sBv$`vI%R?rve#J?g!fE`5Gx!~6aSrEk z0T*$}nvIR#sl_)|#w*0PzdiFj=w{j@g_`(JB%az({>HkYeA(5yp*&txHk4V!hVr+3 z;tkk{P1uZoM{^CX_BxwuE{i(svs;06cR#!7m|v*rfNWcj{yHLpVCbTjdcjW!d1?uN ze&iFX1dao;sN7jE$`ZMohu|W5awDr=&`Yw1AoKrV>fk2psD|pOftsj=+NguNsE7J!fQD#< z#%O}3Xolu!ftF~6)@XyaXovRbfR5;d7<5J#bVWCGM-RlJCwieb`k*iRp+5#-Al||t z48{-)MI4499tjwZ5g3V4cpIZJ24gV}i5QOwn21T3j47CkcQ6g_;yt{N4=^1wFcY&d z8*}g>=3*W`!pE481z3nhSd1lDie*@iPp|@?;xl}Xl~{$wh+l9Bhj9c)aSX?C0>9!U zPT@3u!x{XJvp9$IxPXhegv2=N$so(Ql5PqHxtL@y%2~Iu(dWTP;3?~z|N3He$q*>FeX)l)zqi!tk|9DK z|Ljvsh7$IVBN{V6=clTzt!M>|~kWX5|LqE+Jt_<>L+^mDVp)22YH*`l2#G)s9p*Q-VFZ!WB24EoG z!XOOB5DY~eh9Mr>`0<|~KLn$aK^QxIAQ_bu!q{%9UNAnDLI`@=ih)$3v}AN=0P)}^ zgNT+wuny{=F6yB^8lWK>p)s1EDVm`5_HOLC=Tye&6O{h6P5n7dd-65M$}n#dLH z_mkYc#76cT_EK_BucR0Dn@^XLduBO2}oNp%g zW+eBz)~_g;RTtsjM2Yi1};7Ob@e$noc|!|2?bN@_^>{_@A_dht5{x{|Pnz zyT8J~p>2CGfNI+wZ)Dp;@KACGk51Pm4<&c-7~!H9JW|LV?E4?&YLbU1IfNHqi#L3b z4!%f_3^2hD8R3rrWI|?SK_Ids8-kD>Igk^%5RBZ&gAn9JKIBIz3Lp#x5spGAj3S6Y zQADB`iX#doP!gq38f8!xd1lE&O&wCXXL9fBjV$vNf06`GR>%vH>)7uvdHRU$>^D*>CC>os z8cpMCNS?uR)zsgTo9w)Ym{8Lh;7&&t;8Dn$MDc}>SM zhnjBcp4jU57SyQkS=Ao(la}z%Uo%*@`fuv2Tm7D|Y3O*B%?>SlV3^jjhc}Zwcq!R~ z*XWj#my$hrjp(Ttyq=Ri2)11hawEyhn=Io0dG^47qihiW%Sc`WgEJ+)ZOLnhHH-Kc zeG18Ii8YJhElXa@$s#_nW)V)GK=N8?-Qsmx^z_#xd3|k-HmTla%k%E?^s~OoZOQAT zyi}RAaQwlolg0JPAo*AglHQ^tNII7A$Ppga z`A>_hHAs3Jbb^cQA^PhD?=+4b+NQ^gZF*?=1M6!x=HNrj#XNk3k1-z$un>!|80@;# zW;0mt7jBB`#8@P#ExQ#owi+lJJf8jnJ;GwJwe`FmztOJR+^~kw$riC0< zI9zWZLi9r8wfiq;V8?-@#xUZjCoScP#s(F)p3$Kf?GLi(g@5nmq+S_RCu?iO@fB)o z|e(^)&s9*)zs^)$!oPNg!}+BQEcVmjs$YWl*l zRh&$}O!0*e2UuVD3$~75;Du9pPjBu|>6GD!jyOjdntbd-ljL4Y5d5vzd`PET)}biG zIu!Yc%AG9IzxZPQzxM70E{bw*;P}k41FQl9xhs=kSnX(-aX< z10*FiEig~s5HDGI*F08cW~E|FSy9KlbWyu#rD4|T-utmMD>E@Vzh`E5fJEb*_wCdB z`5ejb@XYMYGc(WgfBrMT>}K|+q5tBu53!1O=0tmY-Z>+hE)LM1iloTW&-+`?G-r(< z-x{TIz4scPY7_eon}nMJHg39h)27lI3vW9T|H3ThMn59ogoiAfA_&0c($O2&L%eOHFBQ!s(k`R)mQni9D1Bj+P8y|ZtyEeO z#JT?E(`h5cy~WY~)^qjJYgzsD`dypYLn0&V$u-<$q*sfjy|QtWNfOs${e4I#`q2+& zDDX!B0%1WD1R)p~O0dm&O(fYCh=7XA(Grn}LMvPWkz}vLRk#{%E7Gz*3hT&EW#|Yepkr;(ch$K4(*~mdI#$p`C<96gB9}`fBJ1`NGP=q@%8B;J7 z(=Z)(;~o^F1T%0iWzNO66_}cNR|L^GHq(4Bb!lbZIs#@rFf&1 zY?N*^N<)lNmQgA&N_QEh`;5{eqqNc}J!X`)7^N4C(jKGqj!`;ll)f@br;O6CTInxO zt$KP-))MTJuGX^4>ipWpd0V~P*Vixay^sv5ly2IzKGb&)8%~ye-$I{Rii4VvVe7on z(~rJ1LxDd65C{vJpece7f@TOs7{YNGn!^ekT0q6+Xo*Nfp%tz`G+N_ITn#%MXoI%6 z2G^n;+M@$v5Q|QTLuYhBS2z)m1aw1pB%%j;;yNV3jb!vfD$SitqW}|d2PR??if|_; zV+y9?E=!|7)$T~9>hafie*@i6kM!4_;qIkw|jcu;}o@H}?l1-yt#yo^`y z5A4LN*oEELgS~hiZ{SVr!+spVTX-Aq;9VTTdw3ro;4qHhLsa1?KElWN1pmaR_za)p z3w(*M@O5Jn>?v0K0YBk1e#RO6g0rZ`IsA$mNS1*8L-@>q5O10dHj+|=QM$?~b+n|3 zp|U~0({)*{^KO-5tkK^n4bw_QW&%6NwuFe-^ln&aY&hLmeYR1$-zcp%N{<_*ZAR%O zqx70lI%t%R8KrM5X?#Y=4*i&a^zAxOe}~?Z4C|?2@|^D%R?f7xO~>VD=4N+rbsS%q z8{wUa6*DBf*W1O8=Ud$$+Y2l_^2L3o)RY(WP2xndsu&s zE3exgnaWz(uT`7buR&WXYa{RE#rZv^h{_gFS?#fZS0URgYO5*ma~AkbX603`Qc=?i zYSL?958{Xx^jIb;Sriqm@3rnhc5%vzWlegU_7x>X+9oQk;DW&QMY)5lNi(X5(3Ot%U%b$0=$%q*?Wil8@48>Xw@0{*}l5m4{uo**kJT<22DmjpcUH!ztt>OII(@@^V>NmkpZB@+6RM5PWHk$aE@5^@9 zV|=FF4BsD;$%cNH||b0Sfmt_BtGm-AElkTh+!HPDs^T2Nlr0|7n1}r zaXCX7Z(?Lhox~(gOgb~6tr{G63^}Q3SLu%j9S@_YMGh(RBxPjo(hYVvkZ!CKre_C=z0Ev7Z>*b;ChiLPRn%E+O7wu71GQ zgm99P&&a36e)h}NB*woz(wE6oPaDIX9u>)Hp8B&$6kL6bYd5nOOm z2dd3dKf%XbaE_Tc)MBcb-=pIE0Yr+__Fg9)R@xPj#<_lpR3lDBRD1r_$CV+DsNU5_ z3rulDj|B3s)gyt%l_&M)^RGNZPt&?N1%9am)N$g<#t3RAMo^7*?V*VZz4cPLf*zEt6kGYt9h?S-UHJv zx=NYeaOYi)d^Id75Q+O}FYC6Z?X8E*l(mmwbY3)xf7iOpU~P<+dT+V3zmO!E>fCH~ z^4@I4iF>Ms?zn1iw$!Q8#qPC8|AiOK@8SmRyu@#hQMZWU-k6&$I!$Y+p4w&9uDa4$ zQduEum5K;vu+=;#PviPIPTZTZWS%RYP}1X^pr66A)OVwzQxB5(4&gn#uiYOK4t1`j z+<2UKyC>F|r+4^;@AZ6r#gSd~Ue6eMfAZ~K&!hg6|BX974*M9V>Al_4$D4kym-_Za z@AY<*S$vK5XW#2xx?KI?ohU(n>2h`Ha&_r)b?I`&Rq)c~>i55z_)9KV`y#C0@tI(U zeb1CRn_u+KZ9`B}^zVPVQZ4+#KRl&Logc#g+kd!9sQs;BXK^EIHCrRBk$w3lh4XIE z_Tob`4{X(@+qh-kWwUw$+gne@T0`~Q=eNV#nhtD>Fdf*+itC#eE##s363^EB^88i# zEvr07Jo}xi@~iTyay3nulnt4IKoXg|W{kaiN zP+A1$d-k*GD&-;G&08GEp~yMr@)Bi$l%v(Y-?M_3%aH}AQf$t5vTdHswxOOQPF79l z!*%kgc#nsQ_w(Y44vG?^9g(Pdff657b=XtFi$|2!Qn^-bPSrkMI-;Z%jF!zU^E+nA z1FZawzofcyZ{>;VW2mga!7xkEiEpb9DB+QOt2{Bidv&lSltT4^D7(3`T#mQcWmV}Y z+k;xMRI$jz1N&81TC;g;aJXf7V9B?~gPX_+a<(-}I>3${=5M=)X%$RA9yMRA#BbO~ zNXOX?J1kePH&>pJs3@j+CVmS9iJE>J~c-i##<-)|OenuNfuZ=A8)>Gowezoq7#=*(9~>@X`&V zixI!|b6Ygd~9*JX6}sHv7y6E5#~=*a*2g%+2n-(HI;l9SE~X zl{q72bIA!hx_FcOPM=L_yifNSv}sB_n}V88k?P;X$_aG zX1giEG;WM!4WeK!%AQ(em-@RRpPxNuT&9)B zMzvQ*-&K@7$$Kz`nMGr3S2P!8=1#Jjtp0rysjI6w!kjm5QjyiG&Gq&>~0^%I4Cfkf?+n!k_+bd2nwlssWugbge~Dre%3Pvt{R7b|&3sB;*ve>UtrLCWsJHae z&t%)DZpZ&TXHIn1{l9h8c9w0%C{a;ue}(A5pog(u%jsRRZRsr8vG1%oQGf4QfcG6| zlJOawwM#A-3EnW%hk;)6E+Wkn?)daKg;4W7wq9Ghz_aePM_)hGO!}c~14^&|=)JP- zOn_tg68}W~!SV(lFZb4GK0kD|Ce_EIUSXmw$+38)f1-F~*}IwO4oS437V_Sh7aO+K z2d<(GSbg)a-6WgN~KZty4G8rS&VtTwh!O;vn?Lv zxbIWFwhRB|)mwWAweLA-I=^~tgeGX|LrWO}-c-SE&fR2N*Va}R=GeDA!13VrfJn~V zxbCU#Yr`{?1eItzlPlWcTxT`yu)N7!qn_)`IoQ725kp>a1lpGG=HZhI=A2`*WyiYR z0V3=0%SiD>D0uUZmUYfs>g$8XWSyNSTHCgG{2T|~33SxF85rq2c`56RX()+G^|MaT z+N`tXAua2ilovN9>pVR(=zp7aW|MUm?+Wr}oo3#wQ~SxkwM#A-2^Y>fOJ5KEQ?t&# zD(bKEHWos-&YbUn8=G}@eHQ$mW}Wq9ows{B?)#h?erzD?@TonvN42c;qe&|JqpWj| ztg~T|CY%AN^W`{Yp{S@Lj`(wx-WSx^*);XhPoz3QR z>KQHTY&@%F9WsjTlUK+)kr6@eL4*2dbklF`B0lkt5no;0z1(j1K!~qCZub;-a;`hg z?ap=g%GH)7C%fG@iV`bcFTOMH`_A0$V$o1mcgML?sbLB)Cr=SI4iNKhF`uH}$tlU&uCv4|tjL$x#(lGxX!E@XjUCom+r4}6fcTi0*zxURViI^YHBDQS zgB3GkqJN3{k_*SA-%_N__a4mpt|`X;25IYS6McW;RP+)j&6n79a?!AiL4`v_Rei+v z?Q(tFCl3`D}O&T57Z|rEdn^VXY@u8oJ2#Mat_Pw?3MKqni{gkZyyz%%^C~svU4RX6jK)J6VDi-RR;fO}A~Ei^Ocrbwo=Obe~DB~6<)RFW;UP?oZ!WPNR-vX-(%g^=V~N<3Sk z2#LtU%aVOpLPD?kpL6fboFd!%d!F}uGu7$Nx%ZyCoX_`j&iUSRzaMv@rzy_F7ueth zNNp<6%>axTrxl}VwiCfxv z-qWO$ox;mFCp(7ZCY?|lx0rLH{ixui5`2gor=;BgqoiVvgA%7$x$o2E$J=x-V;@)S zn!Eg%yv}0wF}Y5`%a1Z}H}+9l8>8h#6owMJsE`)j<_cUw8!!reynDsPbWn@aORRyFx~1A9M6F(26TW?0?;%gbKo11!#2l&95Fs4ykL0Xsv(xc$1dw$ zFwB1;ZaK`CA{x{0m-nyk9gjFSy6^0lcS~a5l#l8&<4NUayNGh1UZHo2<7jWrj_fhH zDu3MZ8D~dypLj9LA-2A3xOw226dk)6Wy4JUkI!e&>dy={9(^$E-m&mA-d#uJjV+0J zeR_zY_ja$%w&AC}^gTD($5CIO8mu=k&0^fKuv4BoE-SPhVqTScba7n7)3FUJb=S0w zr8B6nP7dm9J?GWkqtj0g?9_Yu-Qwt%CkAx1oK%@5oPNSh%`BjJKILVJYx`~^^T!?y zEpfpO24~qv*A+Xr({)YJ7KRo(akT8H(V}b))^uwpg`+*c-?KS4-yEr#ck}&`i1#<% z9Ihe19)2^gM*Q^wTt|K#CD{~wt(RYzIgS9^HWKvzh5!7SbAf@?Tj2R}K6p8> zD>4xq*cKXDBrq^4xdaXD<6N+=oJks3Po=qo3_eq4!0c- zS81=R)}f<%Cyma0O|354I=XrSeFH*{gRStG?F#Y=pLU_70By zot#}<-3AOCmM0~zNJ(9}YW146Y3tUfXJl^JxG8J%maW;_ za<=F0_;u&5-Fx!#_ZIBi|J#9shYlYpEINAZcyY;zlclFlpD8 z>K|3tuHU#>ee3p}yZ7!tc=+h?lc&#~*VMkKd->{h{hNlj@7_25*(Abf|Bf3VuHxt| z*__(x1U6ayJd;;0*r>g^^ri=GC{7aab5<(^yoDjb$5wGM1U^2>1zRBk9>T%$`C$9M z?Z(sp9XI~(LmuPbaRbCKfvoru@;utdU7=G+@bU0Wnh#z?1>{5|Vsdh1O2H>C34-tA zT(DIURowd1J29~=g{_Rg4}8q;bMM917L}3BZcwkj%;w&vdAh{W^__z^#H3PzNv`aH zUjdX|2PvCK#CJs`C%G&A#bpqEqK|UHR&`>^E>-%oEoPw(w*S|1t=#vy>;KxuMqIto z`&*cj=UqBa$GIX`Wq%Gp%$~y(JRca+ehRJx6AU+k-DpcCK|CeEFYx%buD!a#HuazU zQG$v=K7z(8nCT*^m(zYyaHyKa{lm@QqoeUVNl(Yubd*b2KD+jlzYZ%#3D*y$N0 z*JJ&q5ifIf)2}a>*!@NB!I&=U(=@P}4n`T{$iza>?zvr`aid_l23NF@N5oLjTfAIxf+3v%l)1%GyFj4xX? z(ewn(IospfjefMuSTmQU6z;Gy>_(Buw_Z+zlBBIYZ;XX$g}tqolfCxZ)lOEL(O&+I z!dseJjyCb}Q7jL`7w#f^F4sL4C!*DKv2D%IlpFo2Ofd)oX zKLQ4Cd1vWZ7%d*agyd2Sg20IJOoAyfwt+(%Gc?LEH;I=FcoGEs^LA%L^Y4$@&@;n~ z?jp5WtrB*Mwt#neYrKubyxo&+pm_rZ7~uY4Zo(#es_nF+a}8LJ**rV<7@OJ44l~}S z7*Rm&o-U-|#lYMTK=ucTjQu~`&_VWpquSEoVQZj4)b_bTy?wc+-fzNDnqjp z7(HLf)`{kp?N|FGgHjaR;S!TOjAcp}iKfJjk+b$Yrc*tCrt)*mLyz^e+wNvQh@8b{ z$H%id@vGV`d!y&%=H%q&wu^4P-PknANo#|4V@YksgVHhSL5(K|C-<|mRiYl(qAr?(Hvp@LUVym(6zYW4Gi6r)39D}o z2{t*XnABN^fu+zAq^57$ov{7p_TIA>Fn6hgBf&HQ{F)~8=xNr=+`@nl_})<2I+Y&o zA?iXj-COEJHh=s%i<_r=bLXR+7d5VCMNePqB+RfL7oP&V2}((1kH9kqd(2htu;|HP zdv~MgWO}%KqK^0p{VSg?MOhzqLtd^!bx2UuV&1y_IxhHDcQt{|)M zPgFSV&kvgYCV~N0>b8-?xN&ST17!QR9SIqr{>J>}UC(vel0t8nK68f4>llTqf{@Yg zM8F%{XG!$#_ex61`=a>Wbo<`Aw3`$<>Yk9iC@AMKb=pn9}%5!a4 zbei|(&Km#f_qE!N?J_h#*AKxaepAmEncgP-LRe%k(uv$vANE3rQ7?f8C)*Vp=VbnL z?!;99F|eChAnPW!hK|onUiv5g;g6GLmdHt!~$F#urrXvbFE(CGHqU>_&e+zaN`clFBGKN9vWrxB3rR;U{B(|F}otfFoWHX1UvgcOmV; zOEPTPyO1q8p*ea1KS*AAAm*>8vIj7*{ATY%nUf8he!OE!fD{Y(lKFZ!bWe|7bn=`6wug7-KuuA&(pltG#z#)2SS6`-ux-C6r+DrH3-PJu| zqP`*k&By&Honv*rh$^q{ zUW8E^`}`9DL|^T3jqKQD(KxQ*cx}Y3bH~QS^;3v!*QDL(&=6buruy8`F|qxak%~>c zMx6$`+KB3NMPs7-Q6p(h^hQPlt@h2E%f^N8AEy}~dGp6e~x+kbO*Ro8u06VtjDTpls9>)wkl9lPeA>s@7(cg7&iXwS(GBaL<+ zSLkT8tMJ_)hC63AHI-+qFYFx|)^xw|)Z0Zbj0|r2y>Z)H*Ov?L*OXlWUvDa}YK)QYrkTF@x}RiU(MLLWnv^~P#zoNnIIP97CC@jB58(Q z+5i%}lsO>i8-ZPv1F*YAj>K-Ad?z**rU7y=${Zs7DFSAKGx@Ja4W)h0j~lF7+JNYD zS(**&&%Q9MqEO$LS^2&diHskM3_~hm(5H=tcnUz%T8JhhX@(|72NF%Rff6*K98mIF zF`CNdpDANJ0!_4m#3BEEv0A}t)BocGS;i+nYOqon9imSm94R(L-4lrlvqhpVt^|%Q z2=MbD9OZ2QA2Ml%BjyucN0PwtzNWqym`vxmj9PtcO9))3ixFH35do#kGtkCaQ7!zlL6XT12F;G0lRkw`d z!&BhHua$%F#jRi(_4izJwv8Zqv z?XhAb;_(AH2$njbQLKFwY9B?~M}hWHoP88#A4SIq0Z0dNKFHYX&@~P1ZG|2c)8bL^P0w29nS~3K~d21L+LoQjkIl5=cS%C`cX!siPoq6!c&ZNuwZT6eNs-bWxBj z3Q|Quq9{lc1xcbHMHE61<@1KVhfBnpyG;dA$U%bPV~O^hFQDy0&vO_#pgre(y_Fk~ zHWWnDVPY1F(0C*^M5Q5R8YE1EbZL+*jWlR$Z}cbz$pj!t8l*^r1Zj{S4U(flYBWfU z25HfdQZ(eP0v8^Yp2e5eL#qRXyFZ@bCk}&tzm)@p`1Ywf9n?g9>(9) z>j51Q11ix0ahKbnfNAsKJ>5;7?w&j(o`bY=(8vTS=OE!6q??0ebEF*^MI;)8G=q?2 z5K;_6f))>ysE z%}!1^oOv#>_mOU01bIHH z2)bmN+CO8+!n%if8@I7z{Yy8UON|JP9|q(_1u*!MkqS`s!+H{UbJ8!sg0NUvP1wp{hBv-4$B< zF0LaZx}CQe6gWV^xN>h`$-;-%b{fqxF&jI~e7x=NctG{N>#G7*_ZptE{i?oqGOWfX zL%a+`&Ye#KL6$*+;lV^LYJa{KH|Bgj%gK}Dn;eqoPRVy?z!5vNymOvqhUGp5R7Bv3%%C!i*j!w=+2X0^>D&)+}?r^y`})o6mGxa8P}1 z&7GIg?)O%x6qKc`2n?t;zv3`zrb%w`w5e*V;t#Bh-L2)=X4tAR_NCP&OTwQT8sEPP zYbfyO=tR@zF^bOqJniN{z&!I0dMSfTC9qS zGtL}yyYIRj#>(Pbi)LzA(^Zc9%pSLk z4_@NccvlJqFD5(ODG2FPau8o4EG-DBy80-r_xgcbFCN5yi)G)~QL0l8ma+yb_&XiK zf8!OOFUn+5F}hHd5|y9rUVq^TC36!uoSV6e6{ZxOHB7rH$)+wTi+1;Jmn{1#b&JH% z=h3#Im!Ix-=rvDW>zy{Y_sj9hG?UoEhk^z*WBJtM$w{{yBI-6VeFRjUHbZr+)NgsL zD@x{cueRj~ChRv@qv!C}x?#sMm-LJdOOCp3o;NT*HSJi+x&bx=j-5&h6?(m{$X$BC z(krfFZ;olZ`4{~1?{|~q4DV~lKTy6@yJ4`*rerqtH3lfNsc*8W`KovkXvIMnj7Rjt za4Vt}=Y4?*Kj)1gdeC?2B@8UIgx##Q9=mVobI{Y#U`oO5%N4aHVla_%$<#8*oCo+~8T@QEDgMXKInm&aS+hSbsS6~iJA{WZf z5ejst>3B93bW`h?0&4jlRciTe&?}7$t_rNOtO_g$$qPw4YgturwO~m|T1ZuJ)s=#? z1tA4jiS>CQ#Cn(=SXBnHL-MY|-3oR`!u>)z!u`%#68i-b`w`hy!D%47AjFc`&yv_L zBmnMrH4yFwiqa!CAWJtVN(t;3`SksfnmOR_{f#3veIyI-kL;^?3UlBBSh-uW@Cxv%d~pRcHR^H-T#vKU)Z~KhuGG&?K;XHJHHQ z6$o7bdWg|9V!b(G5=3@L!P(#{J+OHiv0q05y9K*LI!1nb#eP-2N;jGVjX-jcK9U6p zho=DJQj7vjBVhb6yU_!u01w>KaGIS8VK;LB3m9E&fv1C+6W5gRr*!y>;p=Glc?#gs zr}XAL3Sf1XfYll?^?e(!4*HN-iBdk|Rm(^*CqNKL?)MMGz#x)OKTV#Ja{`R`W$tWC z0OY*FJUTWbQ=0D-I8FmU?xP#4qTd7ru=RIrMD@X?OL?>pvJ-F+ z#Ffwb^da3c)vyh%Z?z5m6qF?SD$c&z(ULEpZUQ@g#U`HwC)&4T%-QB`Xxadx4b5`E za{)|WvGM&(_zPy8((#oLru$pKlt9Zjb?sU(P$Ypu3DzOxzSy;E(KzFs*#5u%MvDA@{04}z^MUj{sh8~>->Li5A@DVk0yBIu=$dUINDn69kE5>2N>(HR z2cV~bp!abuShqzy2Wz$B*)~|E8rc3ne&;d8pYH~Ut1~*eg?hTyiY$T7o*9)zLl~gq z!HmV{tL+V^l>iO3enII4wlac+oMOo>F62oHu51bnqY7M%`hlIQw+|d zIR;f|lEHm6%isl?X3z-d87TdHH-5zry#Z~0z8fG0&j&n2Ag}jh?YPz9l%a4w?#fPh z0Z`++03RX}iI|Ayw3UKSToMG|$GKpu3p8^6v=cL&g;^-CX!B#i*Y+R0_u`ur^)Q>= zpk95M&Am5S@&Va=}*d zVy5RS4rznkRR!D6L$f~O(X@{ri+SaL@V=v|V`y##nk9-RRb`=BRXe{o>xpYBI<^J1 z6AisuJL76%Aii`y1>XW({81E~1rrR+L4wmQSTa%M=5bym%K+<=crW7rq?_eU=d%pV zthjCaBArg6cO7f76?QYW?{w1NoT7hEZf1XG#QMEA!oAy_dv10oZRU*2wG(e&?fW=a zH+__V!cmDTvQglyvEZM#Tk^;nr-#Qo?7pp>jhAh06V1IUa&Ti#jle)s33ZNbb%NuVEMRyLuC zPEKb`5P*t58=8NA%!ZyBW^@-eR3FWxle)Cp0^a4V@ir1QTlZv}QDnsqaQ`qjVUs=8 zcG}Uo281B0dyLKOWrrDWQ;c9|{#qoMgad`{SMtmq)tZ||us`=1Fn6&8I@GuoEHD9zw7f(hcTd>)V=m8?lp;z< zadM9r3m!YG)wSu9u~7%qdD%7!{wt{TEGTym5)2m*%r29=|5UYqAYu#*oW>;V+fyH~ z``u4eKB!(GQZ0_&C3zV^Qv0CdL6%vs1T%9JerBwhOLrz<_>JYq+GJ;r%J6^M57uF) z$M7E~dy;$8FE$7J4f5Z1YL(vl?oIOsc(0y!B)PJDtCCQj8qP}86bQu%#crc+M>_&p z<9M;mF-i8GM#2PooH%cqzFn^0gNuc98N0Vy;gTl(!~ekkN$Y1ck|UsW*j^~z6qOkBh$lvzp^SQdE%Qp z&PlcNW^ykUyDCei$6=Vh`5St^u-V~j9TK0SoejExNo3CssMt@Av9%|~$oHpc{=Vzw zqcL#OVqG)^ZvY**xnOxN;Cgj}>zT4%eDlheECUe8qPl()DN63#xBQnyb$tLE?wHehpDEovd8WJI*9@%~e+4)=1MM{!oY{LCMGrD4d9API=_Vem37p+mP35^ZEIQ45 zb7zfz_4`_F$95SSG5#!a0GI>@fbpApzR2`8=@-I+MG00qk>fmluxsWp>LoVh4Q!l~ z`O~=*R{>JE0zy?4fL3Aw9O#2YHqig90RW2lV4(jC(Ne6_Y{P`_8K{s0{qs>BC(jod zht_l*=aMAKdEQO=ZLW{!IiCN$CO1uQ@Xby0Uh?b(4`!u5c>8i_eCipRAYRB)twuYN_b%dIQDXN77}W>mE<| z6VAt4bSc|9L#7k+*=C`&{tsv>cxXTYy#}PqXq_g`#EyN~RLq&8-@K_9pc!;8&9?mX zy=fyST?!eVa|^T-vzq3D+&WCbpIsW`%T`S^JwbEM_PBPV9}T(?q)mlyy_^OmNn3l~ z7z@)1ds{6hd+oKWovbvYz5E-6w=}gJZQ|pjSRRHi+(q_WuATP${_6g04bC)%wMx^Q z@lEyYrr8JHy*l6cQt->HJ&%?JJ&K&^WK7l6qEqLyws4gdi!ZF%+6Y4z$(+MqH@Jl8 z0BDfu#C)?2wPsh%^M10i#GB^BM{z%q^dy`E;ft_j2af74okwMU+XSU;6@zIDZd+>3 zS#4cDTgk4r7_pMWQKjW8|#;j_Fj-pQ-#@^Uz~G?Y6s_4@n= zyQ1s4PFqsw>$x?hmNf7=fXh4zpSwFhVx5#eWF7^XN1@qR^6$i+%%dQiqwvk6ApjbgN8#&R*1yAk zrESGe;a`$@6u#Mpz66S&tAb@71(`>oCE@-1JqkA_;Q3$*dmxyd5(p-+tIFPSX<5U~ z847k2rS1f=c3FTL5CaLNE4RHmto}Ic* zf?V9~clug_m6_VYQCjew_H5TI;+^((0%Gba^`USr=l@Ri2Dsx6OHQ0PboMH%U! zbU;EVI!FnHe905M&586jka8Fj+8K(JcED%=X>K6(FeJ7Ui_~_OBDtM4KPEK9e?Ah^ zKN}i^R6vRRgtq)&g?1EKEYdexganROB88(3Na83LX&miCB1gxN%F$oZ7yV~%EfRA3 zr^3$sx36C+?h`Xl8qg4^`v|g#fQCmv>~!$f(G1Xf!#@!57sQ&u1hn2r=xPQTt7$#9 zgN9YoTMAeGHOeMW2^^xQHc*oabz6GMw3qJ5yQ_P`M14g7$Vq|;$aakLgz??~ss;8T z4~5jS9Fbg>2hz*(L4sL9NHHrENoM^F(Ubn$FqTNu$suhzE)u8fgw*K-NS+Raf%qAE z2qXp%e4=qFlf$!s1%kIhF)|^deL))(K^r`Xi9pXWQ7q4U2kQy~+c2OJf)(%|QZW6? z(fG)Mz|m~KXpX=@O?v6xHPC@UZ0a=eJs?im1*rT+BxIa2CO9ihemp6l&B8#8vJY~= z>I)2#RA$JHQ^I~e89mBp!85Xb4%fy+S;?&ovp&E zcM0v7O6`rjGEIcGbe6KZh3~HFi9?oSM=3N0ONq_KYz!W6o1i~I1U@y-Ckxy&tio^S zI}g@%TV*t4qlL4lw$n28?*92s9$NMbm2fQ!CwIP0WaFz_dHn}zSVcUo++f~+pnA_K z*Yf@H90zo;2r5g_G%3exck4P4zKD9E8-4=>m%J?ps+CBZRjcA~vRdUQNU9a(fNH%hM^@_=1)PI< z@j(tIKY>UmBH_nyr2H85Pj`*Lx&7CJpep`dpJA}#6^BPZ%hGIEf0n@7`>Gj5bt2K2 zN>RV5_Jp?Auifw~0BqMF*odSV*xK|b!KOGu0ydNbioPZWTZ+P;Y)qR^9Bl-VA`#i2 zdBg`!Rp~!I>}&IVK8~=~+w_lq+ze?&7QSyqBICy*Q3HvzJ^FYxh@VV|0})9xq$y)0 z(%LvlkcM(V(RJczn%?rAl(9;6;%H7p>WtKGy^!4P7^HXm^YUKX|2Xzww+d6nqW5NT zYM%r%f#xYSWF#ec-U%=GvfaD(_#Yt1Z4(S~OC-(FA|lbEOp&5Rnghyg5(l?^M+Z8A zciF^YC`5|XWR;MdY zR~!H$KM^UY+-BckrHXyznDQtLC_GE>AzrvU}R5#{}e~B~Ws#`|!;VJOp z*K33DhPGf6B{)0-DTZ$ZtN`33iIf&P*W9ny0{6se2xI=aF+B`Q}m7W)!p; zMXE=E>QS6}6s8_UsYgNTi5T^S!v*;pkh=kS8<4XB`5LVI*@!O`^$t+y0QC(}*8q)( z(0~ZJijk)nIf{{=7`cg&ml!#TuU!2@e4%&l-VWUvD-uKJ;Sxjd%$FFtfuY3EVMyiPVq8cJ-PLX2Aa^k?hL4a8 z%+QlKiJ?c#oIOVpqCHL;p&bQiPg}QMjEgN>v&FdBy(dqM3*-blS$ay0iwhSk#kjbB z79nM{7%8JIkuut9DWh$YGTN_FM*B_5XvI=SJ11o{iJ{-VBQf-6&n1T5@K$2z zG&)ntXdEe{sY@A6Ps(VLAe+7W^p!H2la$d0OBrpnl+h+g87)l8Xz1xrqLk56rHqy- zWwh;5Mk|mq+EFQ^l}Q=xvXs$oOBwB%l+hYUMni_aueqVe#Sn&`*d1S|Ky)}qlO4`= zRH8W_-J&&TBceC<5t3AaNYzjYrZrNErtS;~-reaGiul zA2{Np%Ow(xpt6fz=H!6(DRC&i4)#=;xFbw3LjM``LLEP={rsca5CamTBYBrtnARe7 zl}w!pG=A^MIrON2WXqxTlP3=0MIT}mXR2neKeg#xqJi^O#do>c0XcU1e|lLj>;Jwi zQTx%uQyTNkl7F$^G%qN~`@*K3*#QR2tRK5;%+pQPnCIPT@T^zE!v)K%4c97|3Km$s z%Z(SBF6T#Ks%$phj~(R5=YC+{$bI?a%So(FfId!w`=Fj)f>1t_2ee4<^V*>$A1U z`wK%0ajX8DJcy5J)}M#((g`V&SK&+@wyiA1c4V@AS*ImV)UEslhh9v!iVeyYJkgoc z?$o;wi`yPG=H5Cbb(Y>;Q+s%B^Lo4_Xo#oL6szqkPj{^075f^{+xqfV1Kb0%lVGgO zU7Ie1aWj{*g6S;bR?FoL3vVyH8@A?;nv`~PZkf@CDs)l5ZqK^zt30~>!bRQASsBI( zX?8xryyC?rzE4Jv`PI3mo1sk8Tb_77$YFYqj1$NEY_CYXuRE5zWBlYW25jQ+@%6!hPcGS2C)&%9Eb;n;h^qMHY z<^j)P@3d`Cx=g4)#q8nzt6HO6z^LC>@9bP_I@9m;1)F{r6|6h)N2;d|cw@KIVpUX} zapsuYeb?nMRu~NgKydlV@uoOwF< z$;P?##PDNIqaI!f*BrdS+oI3%L4jkF1_h2YeQUq$0A=NHz3b105r4gn@&*Kq#N?Af2J*^E88viGyz9XM~R%=N$RI_z#6eO zb)IAaibcacBU`#wNK2687q0o3; z3z!lx$`iwk5V9{LV04v$(HAis3BhPd><4u2As7RyAQ%G)^nDq_dhjqGfH)FWGm!a4 zH3(`MVc{v-=ty3(jsE}m21X)A95}l4C>qUB^?o4uBEeE0$pYN%JK$SSa>1Vkcpsv( zUa5Q-Xprv^w9p{e!McE-FX@M?IQwcxe`%wFkd9xmi39fv=v#jFmlktI>c|2QPS>~* zj;tf*_)0LN;j|KdpALU9d^HU(rvQxJrwgd%dsM0AyTMbeNQlvILS;)(4)`imc9g>I z?}OIokTrlr)OUdvjX+1PLO2~$@zrR7j*gfb_-VbiwB#3Svp-dnjMn!l2%&wl{-bLo zULa8CQ8`-(eNZtmE$8B|nD7_FCB*cq1u+3)_$!1W;giUav{qWhRvJaXUj-rNL}My$ zanC}$=~<=`bk>oeGtii)g;YfnruoB|;6oya1pK$}>zcZ+Z0V%zcH=9ERu2%}S1|4W z0x@Md;JKjjY+&PunD7_Fx6tvGu<<<90;9osL?11XZ2yQpTEQBkua!r@Du93$z%gtM z$v#>zz!BMXE4UhINwku|B<_NV{1XJ|&l0Wg8e%{3=7B^hXNgjlKr{wd;a7;R;ML$N zX5_Q?<)DirA(%dr1@IrNfWQ&2%abgCHuL+nZ;|iIQP=7jI66AYckIA@64iLV^vd4@ zj1sUWdvW<~!$Auov%y;0bI`cojNk|p*iyM5B8zpNhKRiXT|r z7FWWls*I_j69UHRD1diiRE>OK3b%x1FoLH9_yz0m+!k}yjX);Iros}(`h|_tQF050 z8SoH{$>G7l)5XW=4!(zs8#}=dr;FG6&k7qiU3@ZA{ldnh4e4QiLDO-d9_MBS%IZ#6 zr3X)#9)?rJCps>8T*$b|l9N}7vQaZq%hkoy!4-qoNY&sq;z}w}e3d`qs49!Q>awg< zSt2=jo?wy6fi4&rkXCC`hsDwY$dX6;JoC`&b3-ct$k$lIEIhoW*0&^UfF}z+4={CFPWSPjpx{g%q% z%M`PUvY1uS>s=^Ea+V*}`lneXXNlb3$||o?K>5w4mjbvp`zATzuVt0&RdRooRrdc* zQCVuB`xut+sufmYf||`DSmhNJTl^(f*$7x=siAJeQL?nJVwGzot9)F1>#Wi+Sf%a{ zs!FM;?t*8;Rky|}wFIj)5Ui3*Z&Yg_SmoL?f>i)zsFW=QyrQnEB=qzgG_*I_GgGZ* z8pDr;&`u~6dVoLpaTN+9gtkFKN1-rCXct7}+1d()1K<+a&>8**-zERSw@EN-IM`fh zCUgKLM1a+{5wJve_+1FU+Y==XgIkh^3NW5wYz)5G+S?L`4uEUGhTsGD5e|U~@!iRD z{0MX6@Io*59>&Hd6Ag`xdxEtNj>MMWWZ@T>kRMo{?bNaTy~Bv_PF`U9UJ=scc@o3_ zIIK$e&5k^&R?lw)IKL6Z;q8Ztw+CnL1@{NLI=GVe6BgS1eiFxJgEaX5fKaEpjj1qrw~;(DAQ?!kvxNp9z0*ZTb+ z@G^A-*a)0704CT!g0u%>KX|*uA54`$)h~2f__(mH0aK@T^$QK162YG_E_C{YDUX!+7j3k^M&qv4Xx&{jPS8Ps2tAzP0t1Y2g&W-A0Z(}01Ej%+NDB$x zNQdjxX^QzQziCgOyl?)mVeI_!jFBkM-Dqjo5_muo+vh72EJVw&Mr< zh#lC8UD%C1*o%GGj{`V}LpY3|a0Ewj3_s&IPT(X?;WW7HGJSW{HcyI zD2p(ZLwQs{MN~p%R6$i#Lv_?ZP1Hhd)InX;Lwz(rLo`BTgrfvC9|JHDkr;$13`R7DU?@JrFbu~CjKnC! zAQqz$hcOt7aTt%!F#%s-A|_!nreG?jVLE1DCT3wa=3p-7VLldMAr@gVmS8ECVL86U zSNIw$@C{aC6;@*n*5X^NLp;`F12$q4zQbm0!B%X;_t=gf@FR9$Cw5^s_FymeVLuMw zAP(U$e!>wP#WDPh<2ZqnIEB+VgR}Sr=Wreu@GCCj5`M#FT)|ab!|(V5f8siBz=WH) zh1LL@)G4AM`~(^v3`UL?i|w3WE`i zAsC9!Fbu;n0wXaBF^I)z#9<7^VjRZfb4kyCi*no}LgzvB!Td)<|@IAKU2mFW~ z*oj@(jXl_leb|o!IEX_yjGu4>M{x{4<2X*>Bu?Qp&fqM5!8x4A1^kMOxP;$u8CP%> z*YG?3z@NB|8!+J}Zs9iW;4bdrJ|5s9{=y?X#uGfnGnnz*-9^yoErwY_YX_P@( zgrOYDqXH_T5-Ot#s-haIqXufC7HXpo>Y^U%qX8PC5gH>LP0$q0&>St$60Oi0Z4iOB zXovRbfR5;d&gg=!=!Wj-fu87v-spqA=!gCofPsj_AVgs>qA>(R@fn6;I7VP3Mj-~V z7>ziL!B~vLczlit_yQ9#36n7eQ!x$GF#|I(3$rl?b1@I|u>cFnV~cc_Jhs^SzvXmy zVN$2NE=zQB9$U)0mti@+#8>zlEAS0gVii_n4c6jYtV2B3V*@r~6TZV{Y{6D+!}r*Z zAMhh~U?+BAH}+sJ_F+E`;2;j+Fn+=j9K|vGjN>?glQ@ObID@nJ1?O-c7w{`C;u3zt zWn95kT*L491ApQ=Zoq__xP{xegS)tg`*?td_zREl7*FsN&tS%LIaUZ^yBx9NQ);6$ z%AhR5P!8o$0TodRl~Dy%Q4Q5m12s_#wNVFkQ4jUe01eRyjS-F}Xo_ZNjuvQ%R%nej zh(KGkLwj^UM|47GbU{~iLwEE*PxL}>^g&4({R}?&AR-;x9bHV?4oAJcAj};JUX$%Uos8N#JI^7xE0v9#4YdhPBEF6H%dTXjU0zdfordKE<1 zE4Yel_#J=XPh7_hm~a!fa2t1U7x!=<5AYCw;SnC=37+B^%y=&MDx4QN&MO@K3R=Ve zP`|)!Hm*OoGNIPE)`fr{U%orzxnGg49wWwG^zD3ah0eYAJ-0DNZBC zX=0{b*W@#g(Qpf7IzTiH5SI-I^)(L^O#{Vc1O3LBBSllBxGa*=Akj2PTsDYOlxT_) zmqk$;ESd(3%LY@rCz$RDG509l7q|u97g*bOTI_8!^)`C9Z*tS`ar23@Z2%?fMp3eEC?)I0Q?gbFrT!wf zK&H1mQ-15hi=#zTw74uf+Y_UCke7PvK>3ovbjjd-$q=$6<#SBQD_gSECz9!j80M9QtoCnbu)T*Glnco$;?vznhq?snXjptuXi)wT$Js$HiGgMgXxOF`-&lS ziMf@psgm((YAXj_A#3J7`^)#DYo@Bwe|IG>&szuGFB@MHH1qe zw33TF(MbNvVU+g8gQ&#Juu?JV0aP%b-4Uc2C5=%Vh9O`nNu;l!IwQf97~Y zFx?PhZm{Ae!DJF*Oq6a4rkg^{O-i=}(=8$97Ny&Q>9!Dao6;S@bVrD}L+P$yx+}!o zrS#IVRDfeC#M0t754CUbi5~_jcr^LrGUfX`(N}J~<(chlTC=F<7Rc1D{dJ(+*~l%B z=}pgfcUv_XfF9Ew_QjM zvTIO;7exWVc)W#%O}o>em!bD27;yeq)3Jsn~&T{#%xzYQ%F8io|sv%p656u@czu9d&Z+vA{$MQ%g|CeqPctnk@=QEpMy!AMr znFHj`5Un(p6dFl6lEWjT66fp>4lnJWrBZ!Z6j9qZc}j_3Z*hd!+RTd)$kuLnR4o>y zXu2=N+<)n~c$>iU)M$EY^nPmO0MhGCdi@?fC0i=v5a}l~7=6MD;U znBfW9$!DpTp6x>}_f-%vcB$^c3ZY+fOv)Q=2~w+ya><8(-Bkj*G9H zaaf|HXde6zpK(}EPv6%WM-QW^hta!-v46EBg6?Y@m>QMk0S;T7i?f|_5LezsP)@38 z9--w~W3V;n9ZM|dJ+YiWsv~?+$>uz}6OQ+i^Xv{ep+I5t5-qntrfQU+P#T zcEa)IXLL#ByoB)*yXnv(ej} zbjEu}XCS0LU)Cb2a}V2aTeekclFD?J$4{&I&M~(3rqRgm(6Ku`NwoMjt&aCP#@1f* zfrevjjpXMTTeF?%y^!@BV{3168rYO9W6N)ju{B26UL08Cu48OX`8fEsfyXhn_WDn} z9Aj&Ia+GkM@h(o|k~+_@JG*dc;^&!4vY>lDW6n?Pp z7+d42xuieLcZ{uBuSK%UevYxVm$IK@Y|U}qn&Y}PE;QmoWA!sfO^jn~?X{=4sl8y> zaow8rifqSqYmTus$Jmeqq$m|EZsHbh;|}iP9`54-9^x-N!ecza zQ#^wi&!I6mn>GK~?TE(N5~e056{gb5fAb+)iBpbt`fq5bVOQQ)J9Q)~j@QyCgR%%i zIh02QR753IMio>=HB?6p)I=@RMjg~eJ=8}7G(;mbMmU0 zr3c$bJ4^TfrXQ6xMDtt@#zi&HQxda_DKWb^yDt&j3Nj@(f@&%z28~dDTGV>9+k#2v7G+KYPFYi zO22ZG5~IFjbvI)RwqhIp9c?tA;=634SEQ!kWI#svArmsg zA6f7bK1NmqARDqH2XZ18aw8A&g5T8De1d$)j{+!&AQVC{3Zn=@P!z>b9HA(Ik|>2w zQ5t1X7GWrd@~D7{sD#R>f~u&7>ZpO5sD;|7gSx1P`e=ZLXoSWHM-wzfGc-pFv_vbk zMjJ$+E!v?yI-nyup)*Gb ziBX6_EJh;^V=xxuFdm;{0=~dROu}SL!BkAcbj-j^%))HU!CcJ4d@R61EW%@{>pR!ef>XjOLP6g zrNz#q0%c-J1&f`vcciq+s|M08A~6V2aKs2ljBwJ5{6%psA22wH?t+s}baQkQI5fgg z4&_k+6;TP5Q3X{|4b@QtHBk$-Q3rKV5B1Ri4bcdV5soHkie_kz7HEl9XpJ_AKwGp! zdvriYbV6rzL05D`cl1C{^g?g+L0|Mke+&@y#2Tk}M{pV^Mx}gqg3}DKg5~9c(`+$~ z+($_HyWi!CzsbA=r+on~iLb8UbVST0_Ye}E7MxBfdFdV2_M5D9vhUGp`Li23)>$Eb zm{6vxpmWydwtpZcWkHu#tY(|vyZv9#b52~qjF;D7oQH&RhF}-u8K0PN~WEkTe_E#-vF00tunD>+Ud^PgO*IIOC&T1wKdwT z|I4w#aSB0<{0^J31zYj|{S-owy9WZ@13_@2I&`95^brJ+UV@&x&TxSNM!3Qa?(l#o zyzl|i;6tQ^1aG86dicN>8ITcv$b`)BM;3g9kC7Dt$cF65ft<*N+{lBx2*fAIhx{mj zf*?rz9fJtO5aI792%W@@?{Be!Fiaeu{Oko`j2N8!OHxxYf-q0)XPKQKEL4K}KTxfy zS!4Ux3c^pysUE8=2*+MtdqFrQW_q)#5`=4Fqc?l2{rV+Y7PaH}yWuS01!89V`+{iD z7PfyNv9chhQJTXG^Vz{cm|o7NqK`6_|MuCD9MX9Hr0zvhsbylxsTcLy#c7qZSMfg- z9R7``4vy{tiNn!7cyWYLFLw|0a`!-Q?H*_Zs!61q5FqHC=_b6x?g8OQ?%)3d1btC) zTf(9ReF^1!;w{%i(AN+wjy6~_2{J~j{4$g9PaDFK|$Y&2?Zo%1w2;n$+_)iQi zI{F73p>NqgAc{FB5S$My=LdEV$xjQ;$HlDnpFbwN!~Wg>L^;U$nH=u8P+6WT7oe4r zW^zGZvjmdWpZu7wvDo$y=>S!&f)_<ToUb%RCINLh{BQXjwh{b5cVGPD%JU+(+e1VCW zgvpqKshEZtn2A}KjX9W$d6!_PR56F7-eIE^#-1?O-c7w{`C;u3ztWn95E z{H}ZnZqfc^FL=Gn3-94R9^fJV!XrG!6Fh|(&!I6mZ*Qg#ZoubDhH}=Dw_3`gmhz~j z!fL6sTB@p+8mOgKYN?A_>aUiDsikphX}Vfkq?T5yrA=ySr&{_+EuB?MSJl!Twe(CW z#m2kEHq-y()6&hz6LLP7kUp_|AjU3VKHuvA;oeYAe34W-$ij1SO2~@j(m@`cms7$gNXcdXMR{E@6h|mZpd?D+Q^g&ds@48t)JqY#5wj7A*BU@XQ#P6-q6 z1tww=CSwYwVj8An24-P4=3p-7VLldMAr?VS2}`jI%kd??!q-@VZ?F<_N?3!n_!jFB zkM-Dqjo5_mumxMO4c}use!!2|ft`?3!XE6!KJ3Q<9K<0U#!rw_!ZG}e<2ZqnIEB+V zgR}Sr=WzkQ;vz2LH(bUQT*bANq=P%W@GkD-0UqKnJi=o2(f`>H~ zxT_&Fy;}NMEd?6N$%o2fO9BZol(fF9ton-TYN?T0YQwu|nAUf7Q{OdEeZ>g1^toD^ zsg{8fzJanj zjTG(52e(d=zdX@Mt(8*-aXIGEVM6$Ou1VLT30Q3qHcf$ck*pjvUB|T*!?)$csSGvQqjztJenLvdk+C zC$(TYqf;?#656 zAlwtZ&>MZw7yZy5127Pgh{9k*V+e-gGYrFUjKE05AQqz$hcOt7aTt%!F#!`X36n7e zQ!x$GF#|I(3v)0R^DrL^un>!|7)!7e%kd??!q-?~aL)RT<-1iYmG4%a{zzK+LAW$q zluF1GjkJ;BAih;4zNd3xPD^t+9rjXAOIIo1d0WkDt--fghj^^V25iJ8e22~0f^GO7 z+wlW_#18DlF6_o0?8AN>z(E|sVc2rOQOd_~94BxRr*Il)a2CJd9M0ocT*M{(hRe8u ztGI^Wl{^s8DaF5w@sKzDg-3XdCwPiyFylEiaDom3MCjoR7Z_lKE8O7$Pk14zIFX*` zeBg@=$Ou1VLT30Q3qD3x1Rxu-BL{LK7jh#H0`Up*AwLSBAc9Z`!6=Ln6h$!!qEgx(G1Pe0xi)BZ4iOBXovRb zfR5;d&gg<}=#CypE=~;O`A7^x6b2(2LogJdVHieWBt{_yu^5dwjKNrp!{?ZQFOXcE zn91|AFdK6)7xOS53$PH2umnr749oE)zQWg7fp4(V;4Gh$t~E&3F2zNuutti z{H*5i3+gNWR7($(k~Nu*sxM9O6j(t{B7fIPN#Dfapl9C_MKa_ZId^D zV25sPeLOVRWjt^N{)KeqQ_Q@p9^6?aBVg{yT8m9Vq7M-Fg z9j_H*Lb8|MrDd`Hi7a+#ytZh4C!S{+=6LNoUOy|OQ_-DTW|k=+myc&aPUK)_*|pnw za&&0l63lE2GfP-^Q@l1Ho*7tY(?u8DMc0QH&j^W?=QNEuS?0hJBQL~O#<>#Pu6rtUocT0Neq5KG%FomC7w#<8UwQU0$UQSxpU znW@^O)x*l#9F>s6;WU@~FCI=jAlE>`Q506z&nqkp<@~(7Lgal(bD3UW8umg;IBrP^ zy1!`zPPf$QVlKNzRv@3Xk;x5+M#H3_JW<+QBF&A9?rB5P+?2JUknFT1yH+H}aq`VW z+UvUu4z!;RF9ZiUCfJ7svu=BU7~tn~t8^DRkjf!7DoQ6u2m5v;Cm}f;s!7NiD11)v zZrXk+H6jv`qjUkjKBb5t0U2$%Anbt}FlY*Lc#H9QD2E#Imm1z)OX$+tZ# zHBasu&($GXpM2H?wwfB9?o(W@YHNbEB(Gf@rBohUZmFmprF`b{5OeSk$mHk6&hwIw zSt{nmG$tRi>VD&pm5)O*A)ESgHX{&Fdp^gP*lI_%yUAiTi zKP)}Hm$FO1Cq66Xs6Z}%ZLf}ftJG%+C0_^lgp}uzE%90BeT8&-Orib%NzfyihjHO+?eWy}ALf>=0 zvcy?^pHy@$snBz5rnO&bIV>%m%CN)9b}DxefNYiazB`qUbf}(9IntrEvu`b4u5lXoX>GSr;f-B ztt8Di@(YA%q&s@~rjhmv(o!cr>KWYWOvIVsGZE`LujxGFK=7G$r`C<>JfriOPG?T8 zJFu?vx>L&YYdR~>%MVAK+0VnB*PN2ywrm^&RY2qxxlN}*b2P=V8V>#ClX$C^Sbrb@x0hRMV(sYFW~4O6otDOEY2=;O zDXsOC(#qw=O(RuW@u^NqYdk48Vw%6)T4yNF7gP=o<>Aij4s<$`otMv0-j_jX#p_me z&M-LjyEsLASbfnz`Eal;q?oNBH;n7FcD1$Ax)Ykn-*yCLXBFfHzjz4Qt%`C4Isb=S zYg~}D&^cT>>n2?nHbFp+hZj-t*<1lAeBx|BR$|7uX1oYC`ikdRBB6n zsf#IUNExig6ghOQQ6gG|C37l?aosc}GObY}Q!Aw@ky7QzWeI3GJ1HU0vh-jjqIJ^C zZR2kX2Cv3~!Q|6a?S5YU4%2B)HeHRQ#S)GV7LpJvZxk!0x$s}irgR93Ds6UQm~_k# zF5UBz*6ZaPKc6W`la)5BG@VrS_+AJ?gOw&}Yxr{edu@;Jg-k8Uxivz5pKh>S`g(7D zyQJG9dDNO^<#wWG*>~6&lWmKVt)a4XYm4>Jtj+vC8VXa})r?h|AzvX~?9a6yiG5Yp zMdP9o`7v*IeybBY2oRx%GhASR5w389J3QbCFMNPB_z-C!!5it29zO6z24sXEG9fek zkp&;&V`N1DvLQQiASZGmH}W7a0`Up*AwLSBAc9Z`!6=L(2tiR4Lve(n1WKY5K1FGi zL0N>M9Ll2tDxwl9qYA2`8mglPYN8fuqYmn#9_ph38ln*zBOFc86wS~aEzlCJ&>C$J zfwpLe_UM3)=!DMbg0AR>?&yJ@=!M?sgTCm8{uqFPh{Pa7VKAaG1Vix|hG95HU?fH% z2C*28IE=wqjKg?*jtTez6EO*sF$GgG4bw3LGcgOZF$Z%o5A(4A3$X}`u>?!849oE) zzQWg7fp4%9tFRhtuomB99pbSb8?X_Z@EtZ|3$|h#zQ=a_fFH2~JFyG9u?Ksx5BqTd z2XP38@e_{VsJn}B%w@+MLGv^Bj^hMQ;uKEf49?;goWprsz^}N7OZW|!aRpa#4Zq_L z{E6$h0TXWG7H;DX?&2Qq;{hJxFFe9yJi${ugBj0RV~rtab4LiEGD@Qi$|4NqP#zUf z5tUFGRZtbxP#rZ;6SYtqbx;@eP#+D@5RK3n;b?-UXolu!ftF~6)@XwWv_(6#M+bC7 zCv-*^bVWCGM-TKwFZ4zq^hH1P#{djOBnBZ0gAt7(7>ds@48t)3BQXjwh{b5cVGPD% z9LD2wOu!eIh)I}?DVU0Bn2s5kiCLJ9Ihc!in2!Zmh(%b8C0L4OSdK686~4v_e1nx( zh1FPtwfGk65RdiPfQ{IM@30wLuoc_zJ+|Wq{D>XciCx%@J=lwV*pCA^h(kDxpKt_6 zaST7>I8NXsPT@4p;4FT@Ih@A@{ECaXgx_!(S8x^A@H_s%pSX@2FySU{;WqBzF7Dwz z9^fJV!XrG!6FkK;cbDR314CK-dv5)oL7;U7q%A)D*t^nOl_%hwda+(R)rUTSqqE@X zEI1;?|GP+$&{c4~{G31iLHuxn4gy5z;S3iTV1z5&;0_OX!V4cD4L(F#Nbp8Fq=yfD zkpUUuhfK%}e`LW&_!wCcfNaQ)9LR}W$c;S6i$HvWe8`UiD2N~wLNE%W2trU4#ZVlf zD1nkFg-=l$Wl$DjD2MWNrYg8rEam7dP#LipjW~?KSd7DX63^%IR{{7p!TLX0hO^X# zNuH&C;VLJdiM)FfCSwYwVj8An24-RwW@8TKVjkvW0TyBr7GnvPVi}g>OMHc|u>#*< zC01cI)?h8Z#X7`eJvLw?HsL#L#ujYFHhhom_yIp+2XTh`;a%kMRUg@eF1>m(MHQ+O%~9i)4a@S;LwfkY7a5Qde#nH(@JAMWgpZLG0mz2z z$bp>5h1|%4ya>c6$cOwWfPx4@Aq1l^iXa3(shq;>4^L#CuL$eNj;*|?)Gr`!rjqV_!ssSJRE(6{}-{M_y7;f@jTG_KgaXV zeA18n9@Hp#yh)i@j^|N&?cDS(ksk8#Jcvjdg~5o%5Ddj<7>3~(fsq&mN3Y^Pd|u%> zJhS7x!qKmAM1bu?x*zZ(c3>xVVK??*FaAR#3&qX+AcXwyx%Gc?$Jk+7hvx z(eQIo@+TH)v^ujLkB6TYAH_RcwTSp(S>6J9TeNJ|nqL;?_j!HdGzHCZ8g71Znjp1Q zNG%1crNU~dh*}DvWQx;>ahjM(&4v8=%wsg%0-2fwlSzm%h5DLr3Z|Pv%uT;B=39d4 zmJoA`(rv+XTZp+$>5gE!BgEXHbXPFl6=LpEx+j?K2{HF5-50n8@&^?Qs70Ybe)CX~ z+vcG>s+0nUif>+27g3N!nC=TP_p?>@uxwx;6%#F*qQzy=A?eM7Mblt$*3$8YG$qiOU92iWE(e;<89e14Yw7aoIph14PpRaoGS$PmHD~M(-y^id;4=>iIKj z=9`|}f|*)hmTR7y?Y(Tdb$*mmTUVADr)oN%wXoHsInqnLMNsZzH1#oh_c3P6=w|M1 zH1#%m_m(%Km(kSA=-tamBOKO1I?K-=*O3Hhq%=wk^i^7*X!d-R+pg&az1Z~Lpqcfq zVBgF%`PCLj3$QglnzXHG+e1yhx9v4*)Ve1rCEL>M8r_>0ZB;hyeiBWzAT@6qDlQwE zZJT>Svvb#*-1U3hX?AXUlbe2z8&#hAX4hXd_ZI`1-ttW5cKVhld-3dG=>k8X5lJA4 zWG`m4eBx;zG$%F=`aaU&wA>`i;>9t97m^oKP&1Pz^guxB*QT0g)JJ7plCM`+y~$O-$CVm3>P<%d9=UlN^d^ITkAYHR zhqJ9-X?`~Z(+wf!My^lXZ6S|lmC}&+#7JZaWctefnav~W5jn`qY`b-!V2fC)V!^9A0WqsZS5KI;Lqk>rKx3Jil&uZHceq}D4I-GP?A>)=aa&hMKR9~lBt8_ z-9gF~=x*Nt6b*I8nuXmc+gl<(MP(t6;R=Gi< z|5DnkMKXJ8G(9zXKQ(5fY}fm3nQYYQaZ~O!-5H{l#_~0XNXiM_rt}F7$?T>5v(*1g zmU~UYOQI>g)k%=8a#NM%teq}m)tjF)rYcd$dB9dV_HCdi{mIpn zX=ShNk=sS7H|d-U1%l1KT5f?%ZS_dXNUW$PftR{c38y@7bEf^eXl08%L^ErRGelv{ zF*G;pITodalf;C?IYcxK5tj||p;;z(LiJ|PPG_E4SCiJ&TkD!F$la`QHECSEHS&4M zbEE0G(fhf4V$$7c>TdMzE*HBQP2G&%-Q;3dqp7RWyQ^G$yEQZ2@=U`VI{UwLo6RGa z4W`Qm@5_cR$-na#rfX3`1RfWf;m5B}Mb#f4FOI9mHc8D%As?Dd~ClFq(Q8y?Yq@S4$%3 zCe^Cc0If!4c^JwT=i+RcpSbcaf^t$#^9U`^8iTDl?^t3v@5xJJ2vN!AJi7s@#7Cjt zGUwUt7ok95^Aat$K&EPy+SR#QyGiD}cQMSD5W z$0ySQYza?pfp!_sY&4mT-sYq;-s{GtQn+5EK3~?Nhr}(Ascp!Of;{SG%XryVrAai+ zsSlo3^W9KR3|`K6FLm2W{Jr^3{^Umayyku6J4?1Jz^64KmbUV)KcPusCx zyyf|7va_TiIRU+&QKzI*o;=Q#vP}2t2QHS_D4I+D!IHo@2!0u+N*67ERjF>y?xTsl%?A8Ted15_I=)_1OVH)T*$)Le&X`# z7A5~kiDdHIRfFlO!TYKqTZs?VRF>Mpc1{6V#EZAF-0kH5(rp5dsIm2YX7h`;9_Qs# zM#tFN>n<>GjIGJnP$YkIjIA*|;uu?VjIHS$V{3M2tBgfd9^=m=&qHGuTGmxwn6TYuue>J3S zJod%W{lret9M`RhduV0h%?2bOib!#twSgi(?6c!xU zt;xSA@ZPV}c3ijiazk}oxAu-dMRHuXmckzja9p=$c3ijSxNeQB*IvA6jk}KP)>8Jv zIF9SqUjJ!U($}q>YcK8Pk3U9miDU%dqf7tN7ec+Xmx7YHR@m`XQ00fI_&C<~*#(2r z|B|nQvd2f1lcuNP@0l6d^UB2XYmw1LZS=>2PNUUoo@<>oT7$F32|5T6p@%bEV1N;> zaDzKM;0Z5$fHe3JX(7QI>5v{i@I?k>gdZ{?GyIVSAK_zUMF6rPJ8~c=av?YJATI*( z3GyL73ZNi@Pzb>&j3Nj@Q4~XQgrWpWq7*(wX_P@(grOYDqXH_T5-Ot#s-haIqXufC z7HXpo>Y^U%qX8PC5gH>LP0$q0&>St$60Oi0Z4iOBXovRbfR5;d&gg=!=!Wj-fu87v z-spqA=!gCofPsj_AVgs>qA>(R@fn6;I7VP3Mj-~V7>ziL!B~vLczlit_yQ9#36n7e zQ!x$GF#|I(3$rl?b1@I|u>cFP2#c`(7?3%jugd$AAuaR3K#2#4_#j^HSc;b$Dj37o_!oW>cP z#VPS=^ucc82 zWf6vQD31!Lh)Sr8DyWKTsE!(_iCU6Pa5O{x}qDpqX&AT7kZ-)`l28DV*mys5`z$h!T)HCGAS`?i(a~=3zz!P zlaldQ)cIkRKH6Ej|2O@ptRb4`axgBcd7hG(T}+AD#o2wKD1nkFg-`MSv>7}4TkQMq zu^s>0{jK499C?hLlm~ech)<9Y`B4A`5rjerMqv~|2#Wp>#O(df(oP{<;>UzEmWk!` zH&&~?q*MBpo0J&!9jm(;Td)<|@b74&0Tth68_jCA1o-DlfWMPPI^xY=juELmWn!tk zBck?!mltV3NPqD5TS>&s2B4KE(Xy}W8|(Y+~5umc)|-GAPqi5 zT1fClI;4jWe31be;fGAf41Z+7NB9_75rAySjvUB|T*!?)$P0c@Tk{F>AwLSBAc9Z` z!6=L(2tiR4Lve(n1WKY5K1FGiL0N>M9Ll2tDxwl9qYA2`8mglPYN8fuqYmn#9_ph3 z8ln*zBOFc86wS~aEzlCJ&>C$JfwpLe_UM3)=!DMbg0AR>?&yJ@=!M?sgTCm8{uqFP zh{Pa7VKAaG1Vix|hG95HU?fH%2C*28IE=wqjKg?*jtTez6EO*sF$GgG4bw3LGcgOZ zF$Z%o5A(4A3$X}`u>?!849oE)zQWg7fp4%9tFRhtuomB99pbSb8?X_Z@EtZ|3$|h# zzQ=a_fFH2~JFyG9u?Ksx5BqTd2XP38@e_{VD30N09LEWq#3`J{8Jxv0IEVANfM0PD zm+%`d;|i|g8h*zg_!HN0118+WE!@T(+{HcI#{)dXUwDMac!H;R1~Z;(^^5lK_t1AR zf{;Kkh+@`+yBdR7STGpHOgk7@bR!bDi+cNKQ&$!Yeq!^~y~Tdrk}C^_ATjpUSqp}u zw#^X?p<DM+c2j?jIQC z{(;enCZU4>5qdbo1qS*Fp@Pws=-|d(B7~!VKpj*^4b(&})J7fDMLpC<12jY;{Ppp&toL_jGtG2wLapesuB9^hePH^?m*0tTS{V%wFDEA#~cVGTraLpj*OZw@T zU;e)D3$7)VJ!#!?!L^KXq^Nhx+uu!WWdzq6$}UQz>VjPB%L&`Hfii7Ka$gZB-N_U# zot2ZjGO;B0v(DOymgMgG7Gc40+CX@8oHo3d(*}*3+%a&II|gpA=@{5X4#bLrn>!r@ z5B7>Dyzl|i;6tQ^1aG86dicN>8ITcv$b`)BM;3g9kC7Dt$cF65ft<*N+{lBx2*fAI zhx{mjf(Sw(1fwvDAOuBG48;+O5-5pM_!Om424xY3awv}qsEA6aj4G&#YDh)LfY2x> zh4)Nw>m|-lSd8G-Pl;RqY;nc{d3j8`S)4?j7?;q;gVTL22>`N3~1JAAG<0U&{IqFO=W0soHnHHw87CkpuT2cCT3wa z=3p-7VLldMAr@gV7*lk7-bfw%jz91xuHyzwxQSc1jXSuDd$^AWc!3dKA?2@7vIjg$dPb_V>|y- z`}2yM)FE8jFJFqJOf1cIzgD|apQ>g%TWKAO3ra_v!lkv$#WJzXZmnK>*e6x9`$vwH z4_qt_e4>0Hd&vi~Jh2|gUL447u2nS}v+ysznFQ-W{GZ=Oy~N*9i}ttPl--zyZ*2I> zrL)zgYw38_O}_k&sYaSl&yEYH(>OUVU7lZ`EnQYi zx75-Tr4$?Q#W}+ap`g=&E#y#HRhJ@SUb_|bPymy z4`;Z*068bP!5vBEgS0#+Asx~qseF)`=j5F55t7RXxp-dA33(BS*InLUgx7?iD2kyt zLQw)GQ3`TSD1)*HLphX31yn>OR7Mq4Lv_?ZP1Hhd)InX;Lwz(vBQ!=hnxH9~p*dQh zC0e5mBG4A?&>kJo5uMN(UC|BQ(E~lv3%$_?ebEmCFc6U#geVL~G=^X(KErT~z(|Zj z3}P`FaTtTK7>~~}0bgJuCSfwBU@E3z24-RwW@8TKVjkvW0TyC0mS8ECVL86USNIw$ zAm@ZtSdBGUi*K;hy6H!gE#~^Cmg|1 z9K+8zjuSYEQ#g$?_yy;19vAQ{F5(h?!)08-HT-VL2m2Gv2e){^ZQR8@+{Xhv#9w%X z$9RILFylEi24`#RcU7aOw_3`gmhz~j!fL6sTB@p+8mOgKYN?A_>aUiDsikphX}Vfk zq?T5yrA=ySr&{_+EuB?MSJl!Twe(CW{o|Ryl8s6$`5?BCbS@}mF>A_#>LjKU~_q9}&q2t^5$L@9iV z(kO#4ltXz`Kt)tSWmG{`R6`BaL@m@t9n?iV)JFp}L}P@b37VoAnxh3;q7_;r0&UR_ z?a=`p(FvW=1zph{JMZw7yZy50}zQph{9k*V+e-gGYrFUjKnC!AQqz$hcOt7 zaTt#Y_yQ9#36n7eQ!x$GF$1$O8*?xh^DrL^un>!|7)!AX%kd??!q-@VZ?F=pum)@K zE!H6(>#+eFu?gQ{3$|h#zQ=a_fFH2~JFyFUuowHV9|v#{hj17_;Ruf5XB@`~oWv=d z#u=Q&FF212_!Sp%3BTbouHY)Jr6i8u;e~f`9}n;lf8h}x;|ZSP8O+caoDVeMr))f| z(cE2)ed*QG$7(6iP)FHca>FNQC%%HQcASUA}pYd!9#AF);D%j-#Acx z`3SZ2xmud3mX@fc)oN+8TH39aj;f_|YUy`FIerUdqh+1%*%s|$`O02weQBa6;Z!e` zlP4PK7{iT!*JulU-dvh9xr(?lVBfxTi?(g@76|Ont*wuzreLtAMiV4Y3dxgTc~V%O z6p<$(T2GCe)KAEtuc((sjPdgqWBt6u(TKwsjKw%VucG5MngUFO0{MgIT8hg3q$dT% z@gdWSdTPYkKAFT>KAz%C%)oR^!&D#7qEj@bIha{??RK6V9oko(*%)S)unIC(9gIg1km!LZ`?x8RzG3S*ra?tP2pCoj5IIDQ{&9c#FK;aiQ@=cWJq! zEUUzLO+dboYJJY?c_f0Tr}&H%Cx@1cI-`%uDn@nI4OpigI!G7b9sK-1L87I*>S#JM zqNbJImR&ovi!O>;#f6TP>uD^jVQC~RocfbzVcTKLdD(PY0_!r>Clfn~mr$MJ*w7`s zl~;0~`lG>eI47iO zKp^WzGaF9x+PPlqQ=BEqTN%T~p4Hph7R$0$l&?}zTUe1fA=TuSo5l9!p`xsnRjsSe zCsPzpGlR7Gm7|Gs=ySQH`7RvfI_V>MqLDf&$9UQ|=q)*EHb}+Y)Z;aOkbJ=Ud1(r9 zBx-`?Nnv?XM4p5MdvWB+$EYU1JSiYg3NoSA57D7ob+ zjkk1GG3*!zmvTt?zj{c?`&neBYMWM%D{FhS9Zz$)|Kjn)19A-{97bVf{k+1$P|nZG zD@5LpG?nS~rC~4RgX5Nbp!=IT*Se)n7jxM)umbt44eZ4hqzRJSQX=h*YpZEP(%h7_ zp^)sfB)e852XZpbLmKS6iw?A(4lhIpIVjl21+#8@fEeKCbE|Y0Ig-jTH7ZIchX?z1 zBxfNx9;#W$8Yz5E@NU|EDK#V#lB0A1zCNW0Apsd}$sp{38Zl@Ja(s*Nc__yk5||p_ zEMbNf;3bxv6)#5_a#mPPAJ4E0I8V+x=ftW;F^kcm@vk0vL|J9&b`f<;fGB4IQlK2I z$-56oWUmGE+) z@mk#v++Dv(pRiQgh{h2_&yJ!u$juk9C__>C@pZ!6|0upW$X1@kW%0btXyW)C-bCYC;|%3$q^I6Ve`UKW`QyH+A$Pa@4kV^q8x%L zgrzE`&~NV$1dR)51yUa%lDrZOD@u7hus;3Wb~W__x&gwA8ki zHj6YCFN7Xdh5M`iryBezbCRE1H(hb-W}x>rlyB#A>!fWYm{$kpC1Y6ofv#SFVVN8) zm7!J`iCYJI%lSE%8|=x_?1fNLeOll|5$GV3gO$Q|u+$sT3xtwAv%d$3lf?mK!wqE# z7bZ9pE`nlhhR>()Y2ltIpvKQ?gNqa3SYnGkGA@SqD%CB8nFaMKy%PF0JC<{=7_&y< zTf(&4VVR0!DT}4-RVFkA^(sFB1*9_V>-H)&bLjm#rJ2KfzQSqdkbh;>%;BiwP&9L> znM1>ne`@*5GKaMedK=!3)+*4kcp-Ei|3ktsUk<(h4cTeazR`KV(fPT>GnFshmCw^s z%UZf`4D}fCB(zba$8_*#*zAfE>8H{UrjKi@xKwdI_F#HjTHE@H^A+h8ytpmxAQV@m z$6|3?tX!Pj77MM{W4E^K2JELcpUJ7L=Tz$CV&1PUjgz~C{nXY#4(_7TIGI*n{th4g zU}*N8>fcu1y3-Dy>eKMKsAg{6ywk`_AX9y7#~iru45Uz@w8o-S*iET}3U8wb?1rt8 z_KRu;NY3$%b}Rd4mpm@-hl6-48@+S7brULlqd*VuD095SLF+E>aMhO zs2L;~)#V+o`ql}*Mcpd*Evov|Ez&V*tPcKA)_4NK(oY(UgEsQbIG?p)oaTY&X$fF9 ze+)i$${#;HE7GGJvm8^WybuLsHgE)9;+T3MptBKz!y@{@fPM;xxew9VzC8!L`L+|b z+y{&6@5)|!on!kyF?S?nV*#NKAGiee0sL5Fy?G*7O zbf2C!3aDcducJcfy#QSTs@PxW4vo-pUI>1DFbjsVCBc>Is9>f){`e!R+Fte7O$B3U zpAk<&y}D^gD>?LzflUoClG*78hp>`E7)fq(XQEM7ZR&Z1*;8P`(bkTW%TI6{&82fh zrf*ccBx{@pTE1*FVIwnrBeZ-zH97)a`xIEy5h&Ylv~D_*K$}C>QcpOoGU7>SiAeX# zw)7PD$kxjao{Xja+z&g&A(qS`2A&`KX)@N*WG|5;EIXF_Z)vh$m&U2oabtX*Lvz34 zMsh0aIhFIA$~%b6v^Ltx{lQCVZN}_BiE}qB!n5!EDV)1u5l-)Q?l{Cr`NT(54UX2) z%(W3|nGO$Wl^J|iewYcmuW*=uj_>g28H18pXp}S1{H~ysm8h*s)K-Z~EAw3lw84Yw zq!m09RYKfm(g^PZHj^41>4kSM{c$3n!0;EzGjX?&oP@sz8}oG1E6MBroF8o_f&ABa zqVZ%TGO6NZx+cY59j5X`(`-H&(s<(Ig(_DXPlo4-rcZpl(QxA9g(f?JywG^^V4i6D z#K$8|pZIv8$xa|IG@cC46Td%P%zk9P@@O;hZJ4UuG@fWYQ5D%xsfH6DK2-i_JkfZf zVj)c%pxiCx&r6F*Zo>W=A-L?Q-oBx%LUU_8d=4cm2ax z`VFK6-q!=KtAQU)7JXxx)fbA7B!kg5%KOJhlJOrO0%zp2K_KN2J~IhocczH=5@dey z&qKFJLZT9hI~-5xF+p+=IDvD_?Tm_q7w;0u#iM@~-NSR(18I?$=Vda_p;uy_kS93j zM%CpBq|d@HPI5*!mvjIleyl|BNZYLPJ6-q_{9jexOO?N{K%x53dEii1;**K7iLqm& z`e`EkdiRs0Lx*WBDH7FmEUEDWm9FuG|0%7~HJ_hIOw>z5M(RN4ECq4`W*G@fWYQRO38`sm?A^G|%pee`xm^G~$h38qi{{)D2%AO%82*#4aeoY9Sls+SPKc{7xpj+|zzfu-XL!xe7nF20wjlB!hb2#77cz=56x`1*=Vm z$RlZ%KilC>jSa+@K%Equ3o&XxjM@uvfcn`` z@HCIH*Vrlc8LJ>X%>yB6y~abfq8brc!PYblr%IXb-9Xcb> z+7Je@m#pI)l$zLe_;JkM3j)oxvMQLv!0$NSmJq_qKj(C>SO}}>=5))fgzdb;>E5sr zb|y!MbUQ-{YyKx4(skPj+qWM|O>9>i%l}n}^X+?#u�)&iD2h!aN`8 zaK4S-BJ9I@9nN<*CG7TpaJo6L{Jh_BxM5 zNO#{j!pipOk!}}cdw;4&x+N0`JNdpI=X*E;*mq1%yA7;rB1Y=AoEB(ayP z!C8Lvpg{#MoH~vSYO?vG3;g>vz_{zN_XJI=o4~q1Vz}&#W{<|XGrDqKYv8ile zr-9b%pHZ_QoLqkIe??Q zcZ^ut3fQPS#7kQxfYiSjX|91C2VP}0JXh$v@?80J{M0{&%Puc9(km8D{#QH)dYg59 z48P=N;xIjX&PcCY*ad*z$rdBcx3GE%m|n)pSckoadB!4h{uYF{q#dcN#@XR@9q8Dqz>*kBUo!;m#aT8BUo!-0ZpZ56t2r4sqa-n zsfl&Kj7olEM&T+4u9clNqi{6>!meNBbelm^`!1VN2#&(eRD90qUIobX_i(!Hu#nvY zoUR9$-SBn@(j5e7wC9B&-3uV8*Z(;L>2?9TihmY@^X-M*IDRw)>DB@0y30dwzHVSe zX*E!=vH{4}{96dlcQ1e|=OU+j2zGAoWlpyTvc-SkbaO#GTe~^kg7LuDFF4(6kS*D1 zLApgCshvNu(Ay?<043F#?*=jL*->=_lI|T3Ev$9xalbp>wJ4HWT4GTpwZGD$O6uNU z^HP-5_J4=T3@NFt%@(@raYSDGe_QB>6B)P;Y~N3;v}7VH2ewxqv(lXLtN_&8nK~=2 z4QD&YgCwnGyZ+OvNNUSvo

iw98r8j{e_URY@)U%<3np`#PYz@3LVICAGN7MvEv)suX(_#IqB;z&Ba-DH}b@g>(1EHrgT!XZtxDilG}| zwy({G!ih?L;B#Ku3~;FUl9x8az8=U4rKJXT6os?#x!V&};RLM@E3MxXioyphDtj-K zc37APuwC$eC~Y;d8^EILCA_o=&{lsU6os=Eg|miLvU+RP;F zOQc~)do4LBZSvT~d1J;;#n5hu?M900k}^{!+ZLx`0S;l<^mmmlp#t%f5WSB3AK>8C?h=5 zfX^+kGhKGNp0#`wO5HkwBiZDUI1}ZuF26sC4II}+g4Yoj+bc|YbiO`0)T#XW0|Zp& z&Lwf|@Vib47~Cu5M*xS9-n7$JBWw9}D9zg~`14rBrOf6-R|V#o7YPnSTr8P_?zjUz zbd@md8gu^$>B<}D?z7WefnEJ1j2@7whRHi8qg#Q=8G+U86e7Wah;#6-Ii|aE3KJt! z?kn*d$_J3#FYI(P8>kC|Uo7F1SD+}1iabm)ps~;%3El?4QGFuEmqR!PmJT`E03WTn zK1_w9PJtszpgdTgTVMsINbpeN9Q;`w`3i&h-KoSyf2?=++i47IJ{v|?_-}(O%rfy= zY*r-rIB~Hc9E!N;fbj(Rj`BiS(U*1_!H%5^qu=o(GDX$^1}^zBsr(j!<-8aPE>c`< zzbL1H6Rhc^vcDd}QT)FiK1*qIQ@7QU#k>xap(Moub&Ch@gsocQF8j*5q-WiU{t*XQR z0OeK+$_;!jMY-7(kqTGBX>Kg8`e(i6C13+4S4;uPG$_*T}hgEJ(IXGJBXZ8mw zH*o4ktK81K9S+K^^bhvn$_?DYAj+*5h(xfQPWywDTgRsYYfp#-uQD#SZwTdP1f(3{ z%B@dF@=&=I>I-XzRTrW1|K|LaJja+;!|CTE?25+ekw1Ek9WD2U=AAoHKuT{ zrf~PSShO0NyJol&fWlU&j^BzDm*Tw(?-y`NtFWa!RC07ybTr(=FNo(uCu0pXgaiAy zUd9W*U$lI~nwflf*Go&k7abk*>UX1~XF+X3B5w(k#T&fH2hxkrShsLRCcj_w611Pa z)jQsD-kx9N@o*_}4i*gu4jrGlX65pX)kx|E?El?VwSW9_Yy<0sL+*t+dHm#si7`q1 zcEJ*C{|3%?D-DItkDcmT!0Q2DyeS`#XE!f}sRZK})-8(Pkg{@H*6NkZ zTr;r!Z14CA9`2t%E;v4B&8iJCd^o@#u!v6;F0eSho<)dzIC!PpE@A%A{NQ-WSObk< z(f~~8zg3n8-VfQO9K_+wNZS14*Hbc|PR_`9I%)If*S0#dQZ{ed_}V7tjOo*2ro~9d T;cH=#ll%al?p^9yKK=gycQST$ literal 0 HcmV?d00001 diff --git a/matrix-parquet/src/test/resources/optimized/cdc-1-index-1.parquet b/matrix-parquet/src/test/resources/optimized/cdc-1-index-1.parquet new file mode 100644 index 0000000000000000000000000000000000000000..8ef275c63f64785b26e58dc67a01fb724fb5233a GIT binary patch literal 65539 zcmeHw2|QI>|NcJnG37YfJIZhfQ7DQi6_SjF&^(w*r3uk&9#hG5ks?V*MM=Z02u)H+ zlu9K@C1t2kiT_&rIA^E2-S_>!?|X;eJ^RDi^V)lT*Yo|Zwbx$Hvu%v*v`_@yX^5T^ zRY(1qV40565k&Rm=r2nc;s>RWrnUQoS=}CtzP>5*v}v1>&B$KlOr70z_55>HhSblC;3?n$m8As;i0-GIm!v0X?^`$Zf2%y&|*j-0jOlm#5fXmU7y1`w}*H zf$b$pTZ7wWY;zggGL}(KW$7UOI-62at#g$n!fGitB}|1al^5yK3v4b@B@8Ny$wD$V z#R#eA^RnVw6zbMD@%I+4zk%MHzv@W*-MMZ%(7Ur2u8Oa5aIi*eW?5^*SKCe#L#t;_ zdJ}iYTIYu39ZQWPaaHEZJ0z>7%dU#MJq@*%yggM+Bd*ejCMH=qS53*c2UmI4?4Nuc)+tHDBX?jlS)CdVhWB=~@{gg1Vq1sDMFWFBH5n?MEZm3Tq1<{1r9K9uJ zNC1gUM!G+rTD+~RX8htUttU?{+VmoR{Gts{)}LDFbI)P?LhoBTrxti#LB}sxTlArD z{_0aTy7N7boh+QU;!wQqJonV~g>#qw;-EX%b(c=z9Op1pcg_+&THoR++djX4Ug~>m z(WFi98ZQNP-CQ`)=XLeLslGQC=zBgZ3!!w~m_L5?!veQbpBwXZR@}*$7SvfWckI&Z zamrJDD&}Z8mj=@)o!4iNUi7Q?^OE(~9W~~iaJ>}xq1<7V<6--Q#_P-N)$9(Kgpfa6 zn>Av_o^fs^-q-AuEn=0X1-`#(JKS`;q_VO1RT~B4AS#Xg{>n`G$(!FdUi7*$Lw15! zV_Cqv%htnmS5{{jd0n=W)^;fiCcV34IYe`Qf$K%jOBU$JSs5k)9cAW&RV?F_j6BQC zBowET1NyQgkWY`E9wSL81y6ai`AOG>)~!!IU-13>xa)i?clCVN*4F;j7jPeUbs_Ii zu-eHx^vUO;8&)K-BphmT_>+YLf6C@>9LeA2aFD+-#~$DMVf-Q#cuDU#^a4X2y-6S7 zF;hg33_^m3Q4j>VNhcv>ghZxLX>+rhf<0Ig#fC^p4;>~WD<{uUP#mtL ztTIAXZR99*4b9PGw6w>L)6pHTr$1q$!KBGk42_JZnoKjDZf0&_X=OcQrj4!LEPDsX z*>mR3o4;V;qQy&`oLyYqmM(K&zG9`vs?}@OuJiQrUhm_(VdJLFTefcV^A89N3JwVk z3y+B09<^iVuISw{v2pPUza%Cl?@8IaFEwp{`hkptnTN8na}MVoIeIKF|M-c5lZB^F zpE>*Mx$_r_iZ7OwmR-7hlHU{-m1J^b*H-K?!DUk4<6P%di`+K!JkPwL_z*%wxs?&Z27+f6YcL|02+=Z1O~6~!DKrm zq?StN;z<(Cp{S$9Hvag4P0@{8O`7-7V7f_gL%( zik(FWRn6REe_%Y!+`jQ>1BjI!17qba(MjS+h78PT2c0yXj{I$d9OJ)Z$iHorXW#%p z0!|MEyj>$zQrY}=E_ONxrJ^w=XbIU0os1&v{^q3+>pT@1M+paKyErW^&DPM;fYlkV z28__^uMPF|wTXIh?3nNft>K7Yq8Yj1Gly1rl%b}GTw?pjP}4L?stHJ(60%;%FrS*H zM9SKONd9o2jUq`)(&(ar8I6=R6^T?pl$nOIWE)YEsu6NN?0(>u<^+J+OcI(wQb$jd zP!`c9=FGAtBT7^Z+%u(@P1;5q!U;Eu)K|^El)QONx%uJqdarLxWClrk+1PE@=`x6l zb=q&U93^ zSXs7jO55Hdt=xl8O}meYcRrq99(Q?M!QF#NDiqCFXRoH?Z^itwvN8+9xWwq_4T=FxEBSFj3dZ!S;7>5G`Zx{8W0ydEbqJj?_RZiv9 z8=eJCi}&lLYEvy()e5SiX;dRw#Y?O>B~{6Lhr$6ol!cnvl9o0xh6@9og)G?B);8a6 zjW}RJHePpe%SdK3OLD4Bpy8%n=R-Qw*1>>QRYXFIL7|-;SZJ>_Ptp;)P*gro`Bk)X zcBRaq?fxx!#kv+F<5w-rq1lgJGd>3FQ!@g8u!BiJ5Hgrl%;#g-nwY)wttTj7&-aYkov5d!bx~-G=misJ=ZN5# zwtLTd4b--5jJ_s6)4aEhd7f!f7S3dvT#1-|)*vuABG^PIBBE>}Y4@1+<;KU$4`*-F z)d89Q6hK1XgM!;Kpy0O3hrogx1PV?H795KdLHM%{UT$BPm7YFuV16SvLMsCNX+@0J z)7Bq5ZWISVa+LVwcKn`V1d@*~mVj^5od7oc2~r?c$bq$`L-3-NNHI zSyOF9ceAPMnUM;1fk-o3eCYPqvrKH%17fJ_g(8)0V$w8?@~IX|s=RHPqFOB}vueM1 zf*jQDBcUCj2$KgEVXn$n22v6-co^`z-lBKF?|R&Z?P%+pVR5D_L2c8*QI8X^>V8<^ zJi%Aq=Tl$bz3~qV&%drI3%>a=?+*BVk$8KZ-&jbVU2Iqzpd_J+B^D z$Lwoue7yO((t&&KWj9#T7gW5k_cfZgwP;&sdRrMSj@7@d_44q-^b4(@FSI^+0XAC) zyDVXi)#nIbwsjb9V%>dB8!qVWlInP))%}{( zbLUxcYFOgvT?*apiS6RMu1qyC3=R+XPc$>gWxaHBvfqATD9A1dL2W+>LF;LVJsyH2 z`$Lf5PeKqLjRuCG@cw7Fh;bu1#X-sO6uAsVmtHjTogE&40-bTt%3KO(oIKe+LS1^t z#{vadoMyaul%Zq<<57-+9T&UG5wP^ql(ed?4eituXVSP)jI;A`kjMTl4^sMZ-18$G zA7mg^(n5GSC0_DA$wE)hJUkjf4^126?xM#MK9Eyr!=|zfqjN@*rgEWNF$&zbza&MsZ_Utm zvjBqAq{zy6q|Bg_s+Hx&MZP|im^LR6xMBp1vuT>UB`xb?E!J&~>|ckj$qqRiT%@|k z_=3Tm9haxL`Bi9Xct2eY6`T_UJb1y!n`N)=S8f+|%|r3$K4!BZ;lt)Ad1Rq&K5cuEyKr3#)> z1y8Ahj$c8?ub|@>j&=HncKnu@qTQe;)C6>!5#zr0v*KxpXBlf)-a&oQ(#WQ@b6iKP zsVC~UXKj$S6+sPG|d-BM5C%vMP0$DY|zcRtNchak@{53P#E4p5r zM;X^cpNfe|HF8#yb*$VRir;g0xV$cui3*45)u0NJfZ&*xW z8?x%PIaRgAlOh+R@P2)2lG9N%5QIK&oGKj}MATYz}KMT&>(8 zD4te3uW9M4Txm?ygFiB{E?~=U_yabZ(hiRKH(YEYH|hTqqcR14`Zx7cE(Xv3hKp4* z&|KhR;9dULt+{5zjfynVpm#-*a3}kfo6LVu$gS$LOmxIts%vKvl!h zP*AUR(gxh=E5?n%$lcb2ulNQVkc~(zay&Zxt^Nm_qiW z^iq3h-ITVju6v`4J~!_feWB~Y{L$y%l@A+zuJ!Cg&0k;V?9n{?B6+^%nP-v1G*3U- z^ibo}!xeip3h&LEuW|B@#W0P6TN56tpSV79kNWW|L+7jKmkJM4&nx=$VAQc+$s2lm zZ>H`on&9W%+thRI+dGy}x$^K~hF084wA~mT<+t0Q4a!D7{_%Q8*z?x%)cB&2e)heYJ?0;`cQBr}UQ3NFQt`9uP3TekXxS0g z*m^Z}cafr>NpEP6&_|IDMq}%h)Llh#eg?fhJ+Hd2w}(Azy_~wEXsF+~-laX)y0^D8 zp0!>|-Cl(GsrNeeWOrM(hc&d8rA8F7{gir5dm_3;+8GV4rKw>>gZyO30UNR;I6Y@c zPg`Xr>Rag&*E%Xvr42AB!kJ?8)B#}Q?7%chN*EhPkZ>(4KNC1nRQD_W+Nb*34 z)CheMN5O8-BECs=ay>j6hIB-&oL&;dz^ylQ1h zp@K^ej0?RoXcbKzeMucqbxfgu5k@j3Vb$@HiWfco|BFTi&S(4|vgH2{mke5;(HEfg zmG$ROika-t&&)FX0Ws?&f^2}M2fQrAvG7*2BtRqbA$4&xC?i1T`U6gRD+;etF|a{C zp3J+k3es0MraU`-0=TiGW8o3)E{_h+<2<3ppIbSEvVd#U35?YpnfbDfN(HO`c7XA8|C2!eZh9Hd^ z)v(;Maz^IlM6$`WtQ<0aB8mIt!0&@2$dLp$lAuNs%t(S5N$?_xnwDTi5~N6i6G>1a z2}UGAh$Q%s1RauKLlVzQ1Q(K^LJ~|!f(S|QAOQ`s|ACueff6K8f&)rWKnVsYK>#KA zp9KArV1E+iPlEeNP(KOgCqeuqc%KCAlVE)kq)&qLNl-os#wS7eU9)nUr!jTqRY%KVT<1vqk#nF@h@>WT4q9YR zPCARs^ATa(KY0CML_Y;V9wxZM1a+8T4im&-f;UXih6&a%K^i7F!vtlRUk(7_2dI6(#{xZng8oM3_z zL~w!!PSC&!7C1oyC*BGW6mWt8P7uHe{x?DYCfMHu`J3Q=6Vz{l`Hd65zwwX7-}*;R zg3++6j)!60eU^JwCbqIE?O^$DInLDMH#`uu8GgwQA;Bnk+H0z#mG&?g||2?%upLY#omCLp8RM4Io*ao{QxkCscTPesyk62D=ef!00gXD4@-yzOkZoaJ-Qu;JSTx*TV z!M38hw-*h*Res`RuY1GXGto&au8ert`XM>O*YU)9>v@X@AM*H;lbsYE=Bs-_W!{h;PglXkPuiIRia84mt;{H^x$YaRVqg&tavwK$W$Zy zXJ_TtB^2H*je$2>A=EkUFPx5T+HATV<%ZDp$wyaFE7{7LcubWbB@Zyu9XJ>Y_ z$3!sR28aY*2v(5)!AHG{M>wrK>YvqPFven+AdnGjuaM`WMsgHM-zmfNxmM<`AxJ@u)y@Y6WPlcUWaZk3ssT#h<2`S$&FtYhF; zf6-Rwd2x+VOU!6x`*W<;8%Do0-QInz>1y!P=## zl_I9<^h`(-&RyU#MpTL(c)9mN^5|2v*^X<@9*9^f^~9STwria1`>AK9BXS-S6n@!M znpxZ{=bm3Wm2&YCXTsaFE{j(Pua-?d?Im*3YX4-%@s8osdnP7WmhN5F_-c9KLD_TU zOw8jpHs|@Aj7V2M9bl@usb~}kc+~?RmEVd#7ed(9!)4NjWQg@1>)U(fz6tAC%;5Wv z&JW%%&ARj|{5DnSifXvdzPWollilwoZ?NyoNlFa&^;tUYZrR~q6X#hQysIf2dt7B- z{PygcvJ>eK3s->^2er&z&`F~bd(Bll$!dlHII^nj#Mv9;+jCdDOfWUdZJ7H-islp; zmK3A!Cw_(Tb3`%2T zQCU=G%DCjFI`O#XqmRfob4`-OJ}SwXHjPT~W~SB`A4${D@_@Ri8M=ZYL{F*NXlq6eLuvBr2mO zX|~@$__=$Q>HaY}Y0I5sWF3-%M{El{@$8nB3VrUva&gB?!gkQR1RxZE@J|O}loVPB z!hQ9?P7gY52OSAC#blz@R#|?@kuOyXwrW{D&2^2U+UM?ttWlTBz2iylwlVWvQh)KS zxw*-zVfXc=lT_NnZfstC>fsxoMLH4FHTTT6Zn&MU6h@~`P-Q!~1x>REsb5fju;%t-(B#DU03i`MOZrmeZWF334*RP);W%(2?BAzQR#4{jN8;-#ay zyX&hNS0>zFcxA%1Z8yd{-gFlNx)xICNGmYosakMEo#0y;p#PMQ{~iZ!S&pFzbfAAS z=4&O1B?qZuqDZ05xDlXkI6%%i;<;^I#*?UZvzkh>o#$^Us@IO)wpV<%Q^gk5i6V9B z$|My;6&Z<~2|f1g*zV1`I$I_y>~VavOCwc%_~Fji+da;N9eFnP&HaLA$q++Ul+{e1 z`+51HqZ6j*PIE2VmNifAq1|V*aLcRPWM^6Sq>Q*;Se;pVoc16$Dnt4tO3R(JJkO_e z(5kYL%kMho`FKUzAsiVgQTfA1J5TML{zYjF?Rv52b&csZ!ePIZ?3YNUt#p};`-!QatC5pVq6@OxZH{FHD$UewO&jiAbv>{v2Nt0lEv*kfBApk0;KBOCN2PGPz3OSe%WK87#O9&~ZPv)|+iVW*8%oLXHeW_Yc4cICTmPjb@O=Z%%azhpa$d+P?Sn&@+SC2WB2W1(+B)g3u} zK=FuERtX^;R%*(5BV^;^R;s;v9L*B7HI7v`A04~>&7CE^i-#AcPy$HNPpIRJeMlAo zF&0A+o94hh2NF1)SLf@7*Gsg--=AnvJ6@zmD5d;H{F5Y|$MIA2*Y+-5*V`nrt+8&y zi-(@d3RB70yV~y5q;vD# zMGrgP6w`k&=dF3{xjCMx`Sa|^TISN^=`D`~Ohzl;)(Oh@TqQrw_5I7@Ujnx1OwiX# z*XzofF;>lRZR?{sbEF!4+B$zUe)(MlE?~?KqPq*SqPJO-2aEFq6z)O2jD=N7chORki``YG7p#4S?d66 zj|3>Q@tQmoh?i>E<Gn7-C0=#ET*J($Hq%bkdeV$X`K9RQyjgCGlc}=&~eU9TKk${+C*+;&^Ey0YL*} zDqd+y4<22gL*}aN6@RO;cXFW0o;EJZ#gi)18pC@H$Ja8#CAuQC^@byI8HvcaVjsDT z6r|b^=;aL~+r%K%%HQhk4fZ#t0$r)^_E{)#LKRgL8QC0UJ=0K&nnva7?fI9NdKgY# zZwc_X1qk1Pp4NdHdy{KOKo^h&^2-K7OoM0=obNRDj95@(Z)T)U*OEA(v6p`7D|6|pyyF&5qBmfJt~#59rhr^!4D1_Np{7hnf~k_A1aa{{`j0ck6@dgf;bjU6 z%kgprmI08p^&_;$YC#=23my7ol$w6CL@X<9VXAY>6nJm4&2T1>wbG_{o8hlhW;$nI z`(?ylo!+h1j)_|@#8lseqHJU+D5q-FFF<9rupz({s8)sdt61S`fl!?-f*R7lZJpXw zDMxJioL;Ca_j}O8OAV+SGJX!d@!zN$f&%5rAN~Q*Yw*MS>j&!mwabC-`V21gisT^l zKyAE~)b~;W$bkugq%V4CIK+I4eW(cGm1!9KKHDOp4h#3mR3U~9hH#_?8P`ubiUSoQGnha8KSiQc95@s(N|pE(>P56|^n zFo;>|?0agsO|*jQRVk&bk4WV4-0yyXcPi_dY2fQJnV_?sX(0(a+wl+6L@GzdP$$2a ztT-fQ6oE_ose%_uM)_s?m7b2?G^=gbvf}~w!$t_fmq>9O_`Q%owh8>Ql8kZ$Bc$Pz znj-wriP1vEOKjMMpMifmWH9`64^RqxX_PDxGxN=0kWGQY-@p9r&42j&@;`Z87?*7% zQ26`5^*!5vwomaHbOMDx&^bbI8wrg8f?NFOes}qwjZ+@a`fvE%ZLW*`{T2Q`DWD}_ zU}4ojR`XWIc~PWu_5yWQnuj!{bc|@JY?zZ&DE_x-y!fbj?)yV9;wOD`!I3?p^AQhZ z0g}EyEp^}Cls(BwiN7So$Hm6%j^4F%N7VMni14t`kl>)e0Dr%2Teoc9v~h#4&w6h! z&vk3ptX}1@a>a7@WlP;$U7VekEMBy5!Tfo1XXPKwbgUZLZ0GBnoq6w3Jbgi-`?bL5 zk?|w#GtDB%&m&@0tP)hs0~^D4E1HBdg~*LzyW|XfUS17&7P@2TxTROaNzX#IW9p7s zW&sVs5o{&Xa3-lCC~T07uKQL0r-32Dk}76-j>8Q%)TfbxXu@oc-jXyVfJ7!EQeu5z zdi!>twnJpKs_lNmtsT`7cT}vC#wB90|I#TOzm*#yiZ9k+VS~OO@WIx`p<_2^*|G?1N4oCf#POIgq0-_Zw?#{ z+Q1bz6C#8-2x|OaVT>ra#eX^l+mMCJy(B?QnuKlCZ?=<8ct-B*-nas(iBdCsxAQ3_$;4fW|f-H;v$qy2Qzzrce@keq3#Qr;zDTI&M z^nll2!KZj!zqn$N{xlpuXVuWh-+w87p^%0+@}Bplcsw0KCx}q)SU^a3EF-i#JpXbL zYS2F%P<^lj#gTx*uch2b9w8O>4j_UAN}=_D#nC&E#X+L!2qEuTuny{_;6i%hglGvN z9!7`;aR}9*QG{#|e1}O0BoWdjgoGF&9rTxO;o~Ok^Z$~?YWpl*e~;C{fLrQQEG^Vv zC)pn&ZukFjm4AjFUgJzO@_Wgz4)&uVNQTC3P)=-~ni0e^PwJ~5&jckF<)fJZGl69S z%0_$%EGxbQ)@(kG2^g>9OJHr_Kc*5-sXO@+SV??73qfZ&#+Sf4%a_2q#FxOTZ zR@g=a@`4U(?d?)lV5#yYu!!EH$$SYcbG`(YJzoNAF<$~}1z!Sd zJzoOLpD%&6oiBlvz?Z<<&zHc;L;2!V%6xGuF0%oa=tRCal__7G%9dZn5j7l9!4dTv(EtzUw-ktm0zsb4 z=898+5-s4%M-}nqqssa6Q8j$|sCvG9)N8(cR3}e9is%$N1GVtv;b$KRExf3UId_fb zyB@BAY%_sSBwwLQAQUMOiX12u8KZ+<{a!o4T(ky=&&%%w)xGeAuYd9`)(LjC08wOofjey|$>f_1(tVW81S@&(-)WQm*P6JGwO|bE)F}jp!{i zr_|ZEnm?UOc6EJGV=_Es>muudvy+dhERv2dVSyF(5QUw^X_6e7D6OkfGZZ*aQ=Upv z5&C5*m43lGUCM7L*4d^t-oD*>tz`4+F>XzdmR_vb^r#`oH4$@A(o9};(e>@Z_d4!7 z4{Ut?VHg%!?)H>rwo%D-jnGj0SM(V#jz&@Uv>m={(`7z+k#>9Wq_s|!Lw3`K98g*( zTP7vX^0t+-I8pg2v2y*OZY7T!C@VgdVfyO!ik$UAbF4NnH$+)n7-yPfDqk|#*}>dl z7JTq%8iiJX_v-!wA41*rAP^n zk=eZRaq{$c^)-)gJXm=7@tVU9Ct7X>EepMPUVLBD!}-_C%ga{-p+C{LM20J zYt(bpz4VUWU$gUc#1g5JMRQIK8WO1FkgKjjJ3>RMOo#kH$L?DDLkY)`1<%ixudt0NTMUKlKHN;UX}EE1skLf5 zW6$_6+lrpFo{MC!nW1|uz3lKb^FfM|BQMTLyno-&#yxlaoLz|VT6O;Dg!AdAr{qe8tWB1qsM4dr5G z*`UTvWkOnd9I^6cWlAw`cbY88ZQo@nTI$dj)zG~C!7Be2A)^>`mAQJ_5;5hsPHMH5 z?~+m-Ej#JOy%70L%oPuY`&mzz(h!{lnckfN|om@Y&6k+N~ zo-$m&{krF1)a1y_tR@NHGMeQ)@!-g<_0HQvL(FVmVKo{xk*>AVbTz9R3unsD zIOMo>?ftU{H?6y$>A2%{-uj0OO?!J?u>&?Nr>v*;`BuUw6Lc!cyYtf->?KCCgs;d9 zF-tLF4>x)~vxa0Ve=qNyk*t^qb%CY|S@-fCHd#x!CUL>0*Owg4cJ4oReXX^};VqFd zGiA^hOE|1e9EG+6k!5P~)ebiMpV4g_GcVl95Sp+~yK~5XiD|cHX+4)qUiC!#n2xQo z_rh>wmU%RN*zmZ58P_&VUAfTk5`%4y>^F0djmmzYFJ3#|IA1f(%FQKcK`N3kJiKks zVzJ1PpdJRIblspEH2hmpx_BX^%6b$oN>`EBQ833$JyaBm(k+h0Md{>?J@grYH+5pe zOW4xn2v871twtWp<&@3+NSkeFvuN_3QAG{SjGOLR`YVH;oX{V=D@A@e(@sv`Z~oeC zjbfS`UR?Sp-r8d}lw!!Lydsx1N9d_@_5tzMyz?O&*=K9xZFc2Tnq3cFI2mq}eayD~ zw0B~OeckIj*|%=r%c`1JxWn*d!Ybnf#?~I6y~*g1E^iAL(J-^D{bmI@85v8R2C*m- zXI8V*+E+x4h<5FnvARPeSL-li&>+Uv)MuUf)tb5+kH0)ie?91le`BzT)pEzOz0yN6 z-&K3Ej?7q@wSW4{(~6IOaWdH*w*7Uxs`jjgkxi$iIP>S!_y;TR*)gm{E3Ptj#Gb`1 z>#85-+nyAfCmm=t)bz;o#WZPKGMuF6F!BC2@WaG48sWkrZXX<2pmBFHr_;F$;P5`Y zs~H@X$vY~Oca&-WQOrKP>lHyzva}<9qFN^S;s|n+4u6ZuNCzE}ZkNYz;J1IIyXC`4 zE>25Jvo*%x`Ye!r^Jo1F?&N#h3v42=$-m-{Je+6=yqF)qcs3D4Ho}JlHX&@m^DBW( ze%mI3ctY4lfD^(N0^j-G76P06zD)%2gs_nyo)ESW_|EsX5ZL6mZSqyeh5ldWkwwo4_UloA3hJkLCiL5cuHPM_?0yO?X)N*<64V0Zx8~58gfjP6RmNVc}TpUnk0`5q@IBHbx3%A|_Yg_%_F_k%fAt#h(yE{VB~Y9o4-ME4Pn z#+UnN|K295A;KSp0>+Z`4fy{R>64yA=>uEg|AhWISOzyqj42UD{Dl!8_!CJY66+s^ zg3m~s@fl?XKBEo8XY>l`Io!RQxO=U+dxvuOUdLyQzORR`bkdMa29nG`8tBMNDse3) z;#hdAS&|Nh&;0%Z9BuRaNAZvO`ubqt4EL{sgcN|`GYVkrjw)5e8bR=%ub07TDhQ%x ziukkOpR@!?S_%Pwz;c*BM@_|_gCNpI_~;+A0X7F^AZ^CCc^=p1;4IT9QWS|ZS?zKZ z33)G!|8S5J@?@kQ7-C)j2p;JTBFk4fEpyWtqq$=3vj5gHKUMv&QvHvYfrWfbl}vR_ zbtTn0QB&kBO$n(d!?`Z9kII`%BEE2F}$jOfE5$SFyIpte88`Nq5m^(Oz=4|e1gNP`acB8 zjNu&>!#iqo|541o{_kFWgO5yncvTeze4q>ruj2gn$SUq0>23}$2nt3a5W!Gj6T%j} z3RGYdflYW=_}ToghZDg&T>`m(z0wiXJA%>?#1jGoKeGjIpTLX1x0k>s0-Nys<7e}~ z9!>=969Tz^z0wi1PXwj&vv~4V7m8pk(f3dh*hFBHpSce&;0tgfz=;4S1O|R)3*J6~ z?|g4BflUN9;rYkU<^r4uaPl*J@b(FCBESg`3qP9+a3a9T&+x(9C&0<~H~~XtIyp=# zQ<%#9%~5A`wN69k2C5CyTR52nXaB*tGx`!@Y|giXpD6$Fpfe_d4;>R)j?YBq;WIHa zd?rrnA9N<<&fPngyO*o2B+flke88ZyDjN8_3{uB{f=Rs;;>P(S#Qv3qztEsFEim#( zTJLMKgEih5KRC9JEUg;}#+ofsO912fI=Ms579r`3zwt0L#jl5%DI$-5FwCs%w}+Wk zvh%<&vq~_`EKjT!4l^s^4l`>ewFuV&-&ue$(MC}WL9iB4w26U%F@ji?B$_6K-5i8q z9kOT@1-pqMSeXJE$;5U_gOPtd@>rfaT0w#*^=P1t6znt!!RocpPzrW|j9^XU&}Ir2 zLqV`gJv4=aRZv0V6VVhZc9n);;fCmA3I+y2Vog)g1Pb<&0a7tVGpX1)aOreQnBH9> z1Ph%B)2kCkuqZp2UY-bobvnTGnne*TW-d&xNesb`E`;g%i-Y5wV0xh}kculz?*Ul4 z>JHO;FM(jW9x%O6Yy>;L2Bvp&FoLCc!SupW1Uu~m(+dU}t=C@@n52_~6148dxnNHEP7uvijJf{A9zBG}_tut>$y!-4^B5)R>8YXKly4&z&Er4cMGpA3PJ3ykDbF(rmmH>E%e{$G`V z_g;3Aj9#F_ndXyX|E3G6kyyZkF-^ zKj1y^v-EB#k zx&WBt+)fIF^-X}8v=|D!a5eC^%yv<0Dmg{>l?G&ExmAu2ZLf@+eAL042 z@8t$i(PkV{3E@;USO^1^HuxEvROoL00AC$-ROoIn zOlLOWTdM&MdYbU9)u2H0TBv9yfEskSn^Qi?^V|)DZs2YyZB#TGw@q#*6-}dIm0Wkr z8-<`$=@uPCitgp&?zYMaWrGdMWGV7;R|FLW3si5n4c^L#6)+Wd17oezMvpa4VD}(Y z6eUa1)_N{?aZ?tCQm*O?;4eH9ra|*qh&HTUj5h50q3o9b6xI)Qn}ZoIdAlK^Xk8uwNBpd#DP?Ar)6F!8KqkpfV6a|QdE!S zUfw9V|3=+B+*Y}*=G{EzZz!!UovVV#fE+7>0opK+C@2#Swlk27T~d zoEW0|gRLOa0zp&+6-Ery=wP%z9MX@~<}e^aK2uSDG6F9oDy@UiS<+l1a<3J0h=Y}{ zQ^`|Txv3_j+d)$O>pYW4@oweMf9sA&8vrB$`BG?O1T}y^EU;#PrN?rlfx$aSJLQ1I zFC1o|S#+%83l)tDC-vJLFI*l>xmJ&CAYqj%Dxi)x&_z^Xi&qLZg1;(7%zp_?ms^lc z`3y9Ygw=-7(PVC_gSe34g1sL#B_wR0nhL1$4W#e~2}U1x4(El5Fu-OnMFq4E06KZx6=DgZ^ZTtoyjsHv2I`KLhtqlJBy;nG z`y=;aFa1f_Ig$!!SQtp*tQPJsxX4#)yu_(rL%+O$fvRKWk#yAY>r&vlHaBr?%tQq= zP7HK`uk?ElEKmfR9l8%_o`OW=u(a)TbmUiwsB+^0Hx6)d;YI3e zP7N;$gTSRO-ywf}t>8iidVz%1MbXh>N`FT1IG~3IEIddt`hyd=XwNLHWdxQissh?X zD!m0{8DJpDbCCjs7ZAG%KVX57Xn=JugXZcPD{Wd!%Axj3_lyPED>=nL4X~7GI=UKE z==(K6R8s*}-9<9;+Yv7i_`^jqjCuWR7J#b$s0UH?{Z?L<2gJNyaAi*{ayK7UKZto{ z4IHe*O9iyeRFYqS{Iv<99$$x)*EI#_-$cF1MAXZR$D>{wDH=w-w$lvsEgg%DrK6_- z8H`)g{C3m>t*d`2>OC&Pqh3iW<9DLoCuJ4T*y@jZ;{8z%w3_;(9PfQ)^G_NXb$IKoi_(-E8(WO;BX^tqiG-oU$EQQ7p7wyyKimnXXS~*Rx}!I~Ze|aT*Z1XL&kjEx|KVN_W`(;1G7Tnp zX=Cqsv+dT-g-Kb$^G7Y?oo{Rhj{*4w6Zb5*#J#@Dbki|T_^OFDJboj*Uth4n))vp? zbbKGah%ez;pKhT8?sXH~pbw89ZSp($!|!js>+B6)E--E5P2e28xv(hV+EngM{qX+g zf%}_>U*2SP|MB3??cw?0)M=Jnd%{G&wkLi)FK6u;?#@O=APdVNKDb$dkTBz)Z-$=_ z(e(ZKzAmdgR&cMq%*X`a5AcN_ieC?3^t1uD@-utEeP+h)ZnHP|%$;p(qz#YP=U=}c z58vKD%&_s8d2`&y;FkmV117?675sqA;rTIxxCaxylY88>sXw*{j+eFbz(L@q0WiV& zzFdF6&x7XbpAad|S|`tS>)pIHT-L7DaPstAMLz%l literal 0 HcmV?d00001 diff --git a/matrix-parquet/src/test/resources/optimized/hub-urls.tsv b/matrix-parquet/src/test/resources/optimized/hub-urls.tsv new file mode 100644 index 000000000..a53b9d63c --- /dev/null +++ b/matrix-parquet/src/test/resources/optimized/hub-urls.tsv @@ -0,0 +1,13 @@ +# huggingface_hub=2.2.0 +hf://datasets/owner/repo/train.parquet https://huggingface.co/datasets/owner/repo/resolve/main/train.parquet +hf://datasets/owner/repo@v1/train.parquet https://huggingface.co/datasets/owner/repo/resolve/v1/train.parquet +hf://datasets/owner/repo@feature%2Fbranch/train.parquet https://huggingface.co/datasets/owner/repo/resolve/feature%2Fbranch/train.parquet +hf://datasets/owner/repo@refs/convert/parquet/train.parquet https://huggingface.co/datasets/owner/repo/resolve/refs/convert/parquet/train.parquet +hf://datasets/owner/repo@refs%2Fconvert%2Fparquet/train.parquet https://huggingface.co/datasets/owner/repo/resolve/refs/convert/parquet/train.parquet +hf://datasets/owner/repo@refs/pr/10/train.parquet https://huggingface.co/datasets/owner/repo/resolve/refs/pr/10/train.parquet +hf://datasets/owner/repo@refs/convert/parquet-v2/train.parquet https://huggingface.co/datasets/owner/repo/resolve/refs/convert/parquet-v2/train.parquet +hf://datasets/owner/repo/sub/file@1.parquet https://huggingface.co/datasets/owner/repo/resolve/main/sub/file%401.parquet +hf://datasets/owner/repo@v%2B1/café x.parquet https://huggingface.co/datasets/owner/repo/resolve/v%2B1/caf%C3%A9%20x.parquet +hf://datasets/owner/repo/file%20x.parquet https://huggingface.co/datasets/owner/repo/resolve/main/file%2520x.parquet +hf://datasets/owner/repo@v*/file*.parquet https://huggingface.co/datasets/owner/repo/resolve/v%2A/file%2A.parquet +hf://datasets/owner/repo@feature%252Fbranch/a%2Fb.parquet https://huggingface.co/datasets/owner/repo/resolve/feature%252Fbranch/a%252Fb.parquet diff --git a/matrix-parquet/src/test/resources/optimized/legacy-binary-metadata.parquet b/matrix-parquet/src/test/resources/optimized/legacy-binary-metadata.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d92a0d34d12b2b1615b24436acd794cc8bf89eca GIT binary patch literal 597 zcma)4O-}+b5S?wzstF;6q?>KB2M)U700=}hF&>-(iXVwUG-x<2O9L2?bwPofKg^^5 z#lPXSBp5t7hqm+Hn>W*@4Qv(p7T@QYTh2J708-T?0CeQ}NcDqonVYG_i~~<`CrgSw zZgbl^hD>r(;r_|VVmHlbWtPQ&Z;5OfTzeW0)HvV-Bo3In9R7Jf09` zu=m88;>=;rr{#QE{;j|BpS~hX*8$xEXjlekjwfv4gPuh@F@J~LKxEz2IGDx5MLwL2 z-o`KY%ScUK){)>Qu+_S2pRR^=KiHv;lRi>3^+Iujc!aW#7l?hV7z(A_;OAObTk|zj zN;!o&8vY7#j#NkcuRrA!1flFU39f1CvN-6~uOn3{KX*%uVz*s>P>*sHY8&k*jcHTw z6;dOHSS47SubOII(}?OCUtfhJ+M_N9s=auBYLkU2F;b%hHys7>>zkV8!YIlGaXgu) b-&H&vPF~WbV!n_s2+yPU{k8xa^r3zL^7xY> literal 0 HcmV?d00001 diff --git a/matrix-parquet/src/test/resources/optimized/manifest.json b/matrix-parquet/src/test/resources/optimized/manifest.json new file mode 100644 index 000000000..ef48f0a14 --- /dev/null +++ b/matrix-parquet/src/test/resources/optimized/manifest.json @@ -0,0 +1,186 @@ +{ + "pyarrow": "21.0.0", + "rows": 4096, + "expected": { + "first_id": 0, + "last_id": 4095, + "price_123": "1.23", + "local": "2024-03-31T01:30:00", + "payload_0_hex": "0000ff", + "tags_1": [ + "\u03b1", + null, + "\u03b2" + ] + }, + "files": { + "cdc-0-index-0.parquet": { + "options": { + "compression": "snappy", + "use_dictionary": false, + "data_page_version": "1.0", + "write_page_index": false, + "row_group_size": 2048, + "data_page_size": 4096, + "write_batch_size": 128, + "dictionary_pagesize_limit": 1024, + "use_content_defined_chunking": false + }, + "sha256": "12a973a08998d92a31ea1de7136eeb8d3003a8fe7297a2705f1295231ee4fdc4", + "schema": "id: int64\ntext: string\npayload: binary\nprice: decimal128(12, 2)\nlocal: timestamp[us]\ndate: date32[day]\nnulls: string\nmedia: struct\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" + }, + "cdc-0-index-0-alternate.parquet": { + "options": { + "compression": "zstd", + "use_dictionary": [ + "text", + "tags.list.element" + ], + "data_page_version": "2.0", + "write_page_index": false, + "row_group_size": 2048, + "data_page_size": 4096, + "write_batch_size": 128, + "dictionary_pagesize_limit": 1024, + "use_content_defined_chunking": false + }, + "sha256": "1a956d7b7478b23e2178aa15d74222d85691f67785e6659cc559b3d91738292a", + "schema": "id: int64\ntext: string\npayload: binary\nprice: decimal128(12, 2)\nlocal: timestamp[us]\ndate: date32[day]\nnulls: string\nmedia: struct\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" + }, + "cdc-0-index-1.parquet": { + "options": { + "compression": "snappy", + "use_dictionary": [ + "text", + "tags.list.element" + ], + "data_page_version": "2.0", + "write_page_index": true, + "row_group_size": 2048, + "data_page_size": 4096, + "write_batch_size": 128, + "dictionary_pagesize_limit": 1024, + "use_content_defined_chunking": false + }, + "sha256": "de20b7e5087bfa0d91c8d44618415e1f0b9612c09b75d6872cd07d215eb70d8d", + "schema": "id: int64\ntext: string\npayload: binary\nprice: decimal128(12, 2)\nlocal: timestamp[us]\ndate: date32[day]\nnulls: string\nmedia: struct\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" + }, + "cdc-0-index-1-alternate.parquet": { + "options": { + "compression": "zstd", + "use_dictionary": false, + "data_page_version": "1.0", + "write_page_index": true, + "row_group_size": 2048, + "data_page_size": 4096, + "write_batch_size": 128, + "dictionary_pagesize_limit": 1024, + "use_content_defined_chunking": false + }, + "sha256": "c8cf3a35454a8525e2ef6b9b707fe9e2e8eeab744f5d236ccc5f039a21e6d421", + "schema": "id: int64\ntext: string\npayload: binary\nprice: decimal128(12, 2)\nlocal: timestamp[us]\ndate: date32[day]\nnulls: string\nmedia: struct\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" + }, + "cdc-1-index-0.parquet": { + "options": { + "compression": "zstd", + "use_dictionary": false, + "data_page_version": "2.0", + "write_page_index": false, + "row_group_size": 2048, + "data_page_size": 4096, + "write_batch_size": 128, + "dictionary_pagesize_limit": 1024, + "use_content_defined_chunking": { + "min_chunk_size": 4096, + "max_chunk_size": 16384, + "norm_level": 0 + } + }, + "sha256": "0bb3c02391af5fd082b060d836f00b00ed67b17d238b9715f730e5a45878f79e", + "schema": "id: int64\ntext: string\npayload: binary\nprice: decimal128(12, 2)\nlocal: timestamp[us]\ndate: date32[day]\nnulls: string\nmedia: struct\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" + }, + "cdc-1-index-0-alternate.parquet": { + "options": { + "compression": "snappy", + "use_dictionary": [ + "text", + "tags.list.element" + ], + "data_page_version": "1.0", + "write_page_index": false, + "row_group_size": 2048, + "data_page_size": 4096, + "write_batch_size": 128, + "dictionary_pagesize_limit": 1024, + "use_content_defined_chunking": { + "min_chunk_size": 4096, + "max_chunk_size": 16384, + "norm_level": 0 + } + }, + "sha256": "2c52062818bcf4a8a5ddf329c6401fcaaa2214b4bc5eb7464c78faebb3ee9ba2", + "schema": "id: int64\ntext: string\npayload: binary\nprice: decimal128(12, 2)\nlocal: timestamp[us]\ndate: date32[day]\nnulls: string\nmedia: struct\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" + }, + "cdc-1-index-1.parquet": { + "options": { + "compression": "zstd", + "use_dictionary": [ + "text", + "tags.list.element" + ], + "data_page_version": "1.0", + "write_page_index": true, + "row_group_size": 2048, + "data_page_size": 4096, + "write_batch_size": 128, + "dictionary_pagesize_limit": 1024, + "use_content_defined_chunking": { + "min_chunk_size": 4096, + "max_chunk_size": 16384, + "norm_level": 0 + } + }, + "sha256": "7b08f89d286d43928e4445bb0aa114a2211d1963c7670db5f9a68fa72f7e7bb4", + "schema": "id: int64\ntext: string\npayload: binary\nprice: decimal128(12, 2)\nlocal: timestamp[us]\ndate: date32[day]\nnulls: string\nmedia: struct\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" + }, + "cdc-1-index-1-alternate.parquet": { + "options": { + "compression": "snappy", + "use_dictionary": false, + "data_page_version": "2.0", + "write_page_index": true, + "row_group_size": 2048, + "data_page_size": 4096, + "write_batch_size": 128, + "dictionary_pagesize_limit": 1024, + "use_content_defined_chunking": { + "min_chunk_size": 4096, + "max_chunk_size": 16384, + "norm_level": 0 + } + }, + "sha256": "9061dc6cac1f451ec43aed6ed0fb7392b2d5669b9a5f27ac3aa22a9a187092e3", + "schema": "id: int64\ntext: string\npayload: binary\nprice: decimal128(12, 2)\nlocal: timestamp[us]\ndate: date32[day]\nnulls: string\nmedia: struct\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" + }, + "legacy-binary-metadata.parquet": { + "provenance": "synthetic legacy [B metadata, not claimed as original recoverable bytes", + "sha256": "f46e7f0cc774068aba6cc44575209f92d8efb01ef2c008a403777c532943e8d1", + "expected": "unrecoverable-legacy-binary error" + }, + "matrix-0.7-strings.parquet": { + "provenance": "original writer at b2095159664b34ce426685c387adf0bccdb33e01", + "sha256": "249bb46f28df655b1634a83a92ac3a2942812906136d41677db7291f867f2f75", + "schema": "text: binary\nnested: map\n child 0, nested: struct not null\n child 0, key: binary not null\n child 1, value: binary\n-- schema metadata --\nmatrix.columnTypes: 'java.lang.String,java.util.Map'\nwriter.model.name: 'example'" + }, + "matrix-0.7-date-units.parquet": { + "provenance": "original writer at b2095159664b34ce426685c387adf0bccdb33e01", + "sha256": "7cd86c9165de30d5d5e27f8476db3e37a40eef590c4a2675f173bf2bcbd05af5", + "schema": "value: timestamp[ms, tz=UTC]\n-- schema metadata --\nmatrix.columnTypes: 'java.util.Date'\nwriter.model.name: 'example'" + }, + "matrix-0.7-metadata.parquet": { + "provenance": "original writer at b2095159664b34ce426685c387adf0bccdb33e01", + "sha256": "09dbb3d63b05d58b494b7c3cf92dfb3233cb77daf7ff49a174ff3ffe2230f59d", + "schema": "amount: decimal128(3, 2)\nlocal: timestamp[us, tz=UTC]\n-- schema metadata --\nmatrix.columnTypes: 'java.math.BigDecimal,java.time.LocalDateTime'\nwriter.model.name: 'example'" + } + } +} diff --git a/matrix-parquet/src/test/resources/optimized/matrix-0.7-date-units.parquet b/matrix-parquet/src/test/resources/optimized/matrix-0.7-date-units.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7dc81b31d10278f687f079a31e7992fd32290555 GIT binary patch literal 404 zcmYL`%}T>S5XYz6SObF4(p|C;B$t*7b!mcaS}20xLBvCmo;(yb$wnfZexyllZyr4Q z1fG2&?_R~na4gpDWxkgA{by!$dFN7~ZQ7xqA75|8rUr%;Y67q^0f2J(yNm|k^UKNB z@+#q$xyz!vD^S820JoV*C+Gmi?dv109%Q(=lDS$^0@P>_>tH2yp>@*Q#xAdoqLD z+-?Hmc^4`s|NBD;GuSu@dx_RrOfLZY7)_H+>Zq}8 z)l!*f$y$UnU8K|TX0Ddj@uOTxQRImhzk0!{K4nR+GLgSUGiRl;Wip#Ou5kOpby`7@=*aOy&kLhK_WXetOK%W%gGl=4LF~r;UQc%Vk>B+_ PIXLCpHFQbK# zK4So|u>ltggD{7?{ATg%3;T!C*t(}^;68o)^N|5|li3zq7<9n}XSRSPWUdI-Dy+*4 zY)6=Umw0tF;-fmE%J5Aimq(bKzm*lHSRvPeH)uCz8K{c+v}$isX87;xyVmG!WP4n>h|9?{tCpUCUnmfS6Yjn!MC0$tjg>x zNQ!)}ribSV`qs4??#yo@aUL#z_<<=@@(QAevw_NV?+` g6zCrnl(H+4=j6 zYxFDoYxE6>!IzJGi*VCOW&Uj_;fz2M$|o7Qnc5`4Z5?=AGulEGJ_&%8#BBvvAOChuphD3 z8KL+7tCe0hQR}cn`!wgqV~AfA%697#dLoey2iQ-ITI{Vx_kjJ$G2Je(*W@Y=t88BO zMHCL_PH&{9e$ZKr<;1-HT+D={L@;okU@#bz6UQ_fD(8_HgSqu897Zy9{E;V>69`|L z8*(oEv65gu7>n@bR7SV`ur!n5WH<^+6{pg4Dy5z7bf~;iqgSo=ysoG>TGhU&wtBU$ jCmMIU{Yt-CuZ!KL*QhnBqIHwI7C`#I9)7|H_#gcR6rQ!^ literal 0 HcmV?d00001 From 0d71ce57650903408c8d5b08869c401199d13936 Mon Sep 17 00:00:00 2001 From: Per Nyfelt Date: Sat, 17 Oct 2026 20:24:59 +0200 Subject: [PATCH 2/5] Fix optimized Parquet review regressions --- matrix-parquet/readme.md | 28 ++- matrix-parquet/release.md | 8 +- matrix-parquet/req/optimized-parquet-plan.md | 4 + .../req/optimized-parquet-review.md | 71 ++++++++ .../matrix/parquet/ColumnMaterializer.groovy | 4 +- .../matrix/parquet/DatasetBatches.groovy | 28 ++- .../matrix/parquet/HttpRangeInputFile.groovy | 37 +++- .../se/alipsa/matrix/parquet/HubUrl.groovy | 9 +- .../matrix/parquet/MatrixParquetReader.groovy | 129 +++++-------- .../matrix/parquet/MatrixParquetWriter.groovy | 171 ++++++------------ .../matrix/parquet/ParquetBatches.groovy | 22 ++- .../matrix/parquet/ParquetReadOptions.groovy | 1 + .../matrix/parquet/ParquetWriteOptions.groovy | 17 +- .../src/test/groovy/HubUrlTest.groovy | 10 + .../groovy/ParquetDatasetAndHttpTest.groovy | 50 +++++ .../matrix/parquet/HttpTransportTest.groovy | 26 ++- .../ParquetReviewRegressionTest.groovy | 162 +++++++++++++++++ .../src/test/python/generate_optimized.py | 8 + .../src/test/resources/optimized/README.md | 3 + .../optimized/arrow-all-null-string.parquet | Bin 0 -> 418 bytes .../resources/optimized/arrow-json.parquet | Bin 0 -> 676 bytes .../test/resources/optimized/manifest.json | 10 + 22 files changed, 565 insertions(+), 233 deletions(-) create mode 100644 matrix-parquet/req/optimized-parquet-review.md create mode 100644 matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReviewRegressionTest.groovy create mode 100644 matrix-parquet/src/test/resources/optimized/arrow-all-null-string.parquet create mode 100644 matrix-parquet/src/test/resources/optimized/arrow-json.parquet diff --git a/matrix-parquet/readme.md b/matrix-parquet/readme.md index 6986680f4..f94298946 100644 --- a/matrix-parquet/readme.md +++ b/matrix-parquet/readme.md @@ -74,7 +74,8 @@ ordinal row seeking API. Datasets validate every footer before returning rows and require identical logical fields, order, decimal precision/scale, timestamp units/UTC flags and Matrix type metadata. Duplicate canonical files and empty selections fail. Directory reads include direct `.parquet` children; -globs are explicit and can select deeper paths. Dataset indices are omitted. A source column +globs are explicit and can select deeper paths. Compatible Matrix indices are restored when every shard has the same index metadata and +projection retains its columns. A source column contains canonical absolute paths and must not collide with a physical column. Parallel workers use independent streams and converters, reuse each parsed footer, prefetch at most one batch per worker, and preserve file/group order. `rowGroups` is for one file; use @@ -96,15 +97,19 @@ def hub = MatrixParquetReader.builder().remoteReadMode('AUTO').limit(20) Replace owner/repo/path with a real dataset. DOWNLOAD preserves full-download behavior. AUTO verifies `206 Content-Range` support then uses ranges, or downloads when the initial probe is unsupported. RANGE requires range support. Once decoding starts, transport/object-change errors -propagate. Range caching is bounded (individual fetches capped at 64 KiB). ETag/Last-Modified -validators and total length detect changes when available; same-length changes without validators -cannot be detected reliably. Redirects are bounded and HTTPS downgrades rejected. Connection failures before response headers -are retried once; malformed responses and body failures propagate. Streams are spooled to temporary files; byte arrays use a seekable memory input. +propagate. Range caching is bounded by `rangeCacheBytes`; individual requests are capped separately at +8 MiB. Bulk reads request the needed span rather than splitting it into 64 KiB requests. Small +reads prefetch at most 64 KiB. Validated redirect targets are reused, and fully consumed range +responses permit HTTP keep-alive reuse. ETag/Last-Modified validators and total length detect +changes when available; same-length changes without validators cannot be detected reliably. +Redirects are bounded and HTTPS downgrades rejected. Connection failures before response headers +are retried once; malformed responses and body failures propagate. Streams are spooled to +temporary files; byte arrays use a seekable memory input. Hub dataset URI parsing is pinned to huggingface_hub v2.2.0. Revisions follow the repository: `hf://datasets/owner/repo@revision/path`. Encode ordinary revision slashes as `%2F`; special `refs/convert/` and `refs/pr/N` forms may remain unescaped. `@` inside a file path is literal. -HF URI paths are literal (already-percent-encoded path text is escaped again). `HF_TOKEN` is +Dot and parent path/revision segments are rejected. HF URI paths are literal (already-percent-encoded path text is escaped again). `HF_TOKEN` is optional and read only for explicit hf:// calls; authorization is attached only to HTTPS `huggingface.co` on its default/443 port and never to CDN redirect origins. This is HTTP range access, with no Xet download/deduplication client. @@ -112,7 +117,7 @@ access, with no Xet download/deduplication client. ## Read options Use fluent builder methods, `MatrixParquetReader.builder(typedOptions)`, or SPI maps: -`Matrix.read([columns: ['id'], limit: 10L], file)`. Unknown options are rejected. +`Matrix.read([columns: ['id'], limit: 10L], file)`. Unknown options are rejected. Null map values preserve defaults, including nullable AUTO options. `ParquetReadOptions.describe()` lists these options at runtime. | Option | Default | Meaning | @@ -127,12 +132,12 @@ Use fluent builder methods, `MatrixParquetReader.builder(typedOptions)`, or SPI | rowGroups | null/all | Unique, zero-based original footer IDs, caller order | | rowGroupsByFile | empty/all | Dataset-only canonical file path to ordered group ID list | | maxBatchRows | null | Positive accepted-row batch cap; default one row group | -| sourceFileColumn | null | Optional dataset path column, added after projection | +| sourceFileColumn | null | Optional canonical local file/shard path column, added after projection; unsupported for memory/stream/URL sources | | parallelism | 1 | Positive maximum dataset workers across files and row groups | | remoteReadMode | DOWNLOAD | DOWNLOAD, AUTO or RANGE | | connectTimeoutMillis | 10000 | Positive HTTP connection timeout | | readTimeoutMillis | 60000 | Positive HTTP read timeout | -| rangeCacheBytes | 8388608 | Positive per-stream cache ceiling; actual fetch/cache is at most 64 KiB | +| rangeCacheBytes | 8388608 | Positive per-stream cache ceiling; independent request cap is 8 MiB | ## Writer options and exact types @@ -171,6 +176,11 @@ not enable a filter by themselves. Column settings use physical primitive paths | bloomFilterNDV | empty | Primitive path to positive expected distinct-value count | | sizeStatisticsEnabled | true | Collect size statistics | +Files are written to a temporary sibling and atomically replace the destination after successful +close. Failed writes preserve an existing destination; filesystems without atomic replacement +support fail safely. Existing POSIX permissions are retained. + +JSON and ENUM annotations decode as UTF-8 String; BSON remains binary. Strings are STRING-annotated recursively; byte[] stays raw BINARY. Byte/Short use signed width annotations, UUID uses FIXED[16], LocalTime uses TIME(NANOS), and Instant uses adjusted TIMESTAMP(MICROS). Instants with sub-microsecond precision or int64-microsecond overflow fail; diff --git a/matrix-parquet/release.md b/matrix-parquet/release.md index 5381c3dfd..c707f0572 100644 --- a/matrix-parquet/release.md +++ b/matrix-parquet/release.md @@ -14,6 +14,7 @@ Complete defaults, supported types and end-to-end examples are in [the module gu | New Matrix strings | STRING-annotated BINARY recursively | No opt-out; standard annotation fixes interoperability | | Legacy Matrix plain BINARY text | Remains String, including nested text/map keys | Automatic; metadata carrying the new format marker overrides legacy inference | | 0.7.x byte[] columns | Fail with a contextual unrecoverable-legacy-binary error | The pinned 0.7.x writer fails with InvalidSchemaException for byte[]; legacy `[B` text payloads cannot recover bytes; regenerate from source | +| JSON / ENUM BINARY | UTF-8 String, including dictionary/null-only cases | Preserves pre-0.8 text behavior; BSON remains byte[] | | External raw BINARY | byte[] | `binaryAsString(true)` restores prior UTF-8 behavior | | Unadjusted timestamps | Local wall-clock time preserved | No bug-preserving flag; applications needing the former shift can explicitly convert the local value through UTC into the desired zone | | Integer-backed DECIMAL | BigDecimal with declared scale | No bug-preserving flag; explicit `unscaledValue()` obtains prior raw numeric value | @@ -45,9 +46,14 @@ External INT96/FLOAT16 raise contextual unsupported-type errors. External TIME s java.sql.Time at millisecond precision; exact predicate filtering preserves original precision. Mixed Date/Timestamp policies are now rejected before writing, preventing the prior 1000× unit corruption. New Object/supertype values are resolved consistently for schema and metadata. -Unknown SPI options are now rejected. Dataset reads omit Matrix indices, require matching type +Unknown SPI options are now rejected. Dataset reads restore common compatible Matrix indices, require matching type metadata, and reject empty selections and duplicate paths. +Review fixes also cover empty Arrow dictionaries; consistent static URL/Hub and directory overloads; +local source columns; daemon dataset workers; null option-map defaults; path-segment validation; +projection around legacy `[B` columns; one validated type-policy scan per column; and atomic output +replacement on success. HTTP bulk requests reuse resolved targets and a separately bounded cache. + Native CDC writing remains deferred to upstream parquet-java #3818; the Python rewrite preserves custom metadata. Hadoop stays as a runtime dependency because codec initialization still loads it. diff --git a/matrix-parquet/req/optimized-parquet-plan.md b/matrix-parquet/req/optimized-parquet-plan.md index 4228d2b72..5429af520 100644 --- a/matrix-parquet/req/optimized-parquet-plan.md +++ b/matrix-parquet/req/optimized-parquet-plan.md @@ -780,3 +780,7 @@ Benchmark commands (init-file setup is fully documented in the report): ./gradlew :matrix-parquet:test --tests '*ParquetLayoutBenchmarkTest' \ -I /tmp/parquet-layout.init.gradle --no-configuration-cache --console=plain ``` + + +The subsequent PR review fixes and their verification are recorded in +[PR #481 review fixes](optimized-parquet-review.md). diff --git a/matrix-parquet/req/optimized-parquet-review.md b/matrix-parquet/req/optimized-parquet-review.md new file mode 100644 index 000000000..d8f002b49 --- /dev/null +++ b/matrix-parquet/req/optimized-parquet-review.md @@ -0,0 +1,71 @@ +# 9. PR #481 review fixes + +The numbered tasks correspond to the thirteen findings in the review. The fixes were developed +on `feature/optimized-parquet-review`, based on the original PR commit eee2b16e. + +- 9.1 [x] Handle empty dictionaries without a descending Groovy range. Add the exact three-row + all-null PyArrow string fixture, generator provenance and a read regression. +- 9.2 [x] Decode JSON and ENUM annotations as UTF-8 String, including dictionary conversion; + retain BSON bytes. Test PyArrow JSON and Java ENUM/BSON with binary overrides and update the + migration table. +- 9.3 [x] Separate the 8 MiB request cap from the configured cache ceiling. Bulk reads use the + requested span; reuse validated redirect targets, refresh rejected signed targets through the + original URL, and leave fully consumed successful range connections eligible for keep-alive. + Measure actual request counts with a local redirect/range server. +- 9.4 [x] Route static URL/URL-string overloads through the builder transport, including named + and zoned hf:// overloads. Test bounded timeouts, downgrade rejection and Hub parsing. +- 9.5 [x] Write local files to temporary siblings and atomically replace the destination only + after successful close. Preserve existing POSIX permissions and clean temporary files on + failure. Test rounding and precision failures after a valid row and compare original bytes. +- 9.6 [x] Support sourceFileColumn for local single-file reads and batches. Restore common + compatible indices for both collected and batched dataset reads, including truncated batches; + omit indices when projection removes their columns or shards disagree on index metadata. +- 9.7 [x] Resolve each top-level column's validated type policy once and carry it in WriteSchema + through decimal/schema construction and metadata emission. Remove obsolete duplicate inference + helpers and report full nested logical paths; test nested error context. +- 9.8 [x] Treat null option-map values as absent consistently, preserving Boolean/default/map + values and nullable AUTO semantics. Reject null bloom entries with contextual validation. +- 9.9 [x] Create dataset worker threads through a daemon ThreadFactory. Assert their daemon + status while an iterator remains open; explicit close is still required for prompt cleanup. +- 9.10 [x] Route named/zoned File/Path/string overloads through the shared directory-capable + builder and test directory results, names, time-zone values and index restoration. +- 9.11 [x] Reject literal dot/parent path segments and decoded revision dot/parent segments; + retain the pinned client URL cases. Test nested paths and encoded revision traversal. +- 9.12 [x] Reject unrecoverable legacy [B metadata only for required output/filter columns. + Test successful projection around the damaged column and rejection when a filter needs it. +- 9.13 [x] Correct the reader's memory/batch/cache comments and writer option documentation, + including literal V1 defaults; update source/index/transport behavior in the module guide. + +## Verification + +The required ordered module gates passed: + +```bash +./gradlew :matrix-parquet:codenarcMain :matrix-parquet:codenarcTest --console=plain +./gradlew :matrix-parquet:spotlessCheck --console=plain +PARQUET_PYTHON=/tmp/matrix-parquet-fixtures/bin/python \ + ./gradlew :matrix-parquet:test :matrix-parquet:compileConsumerApi -PrunSlowTests=true --console=plain +./gradlew test -Pheadless=true --console=plain +``` + +The Parquet suite reported **133 tests, 128 passed, zero failed, five intentional opt-in +benchmark/capture skips**. Default-size CDC generation/read and the downstream consumer compile +probe passed. The full repository task passed; unaffected tasks may be up-to-date. + +Additional successful checks: + +```bash +/tmp/matrix-parquet-fixtures/bin/python matrix-parquet/src/test/python/generate_optimized.py +/tmp/matrix-parquet-fixtures/bin/python matrix-parquet/src/test/python/verify_matrix_output.py +``` + +Fixture regeneration is deterministic. Checked-in Parquet binaries total **738,113 bytes**, +below the 4 MiB aggregate and 1 MiB individual limits. Arrow verifies SNAPPY/ZSTD strings, +binary payloads, exact decimals and metadata-preserving CDC rewrites. + +HTTP request-count assertions show one 1 MiB data request after probing, one redirect resolution +per input, and no extra request when the default cache serves the repeated block. A 1 KiB cache +still permits the same single 1 MiB bulk request. The test checks byte-array and ByteBuffer +readFully paths. Successful bodies remain eligible for JDK keep-alive pooling; socket-level +connection reuse and live Hub latency were not measured. Existing selective-transfer, page-index +and bloom-filter byte-reduction tests also pass. diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnMaterializer.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnMaterializer.groovy index 95eabf08a..6e92eb0ca 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnMaterializer.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ColumnMaterializer.groovy @@ -157,13 +157,13 @@ final class ColumnMaterializer extends RecordMaterializer { void addBinary(Binary value) { accept(value) } @Override boolean hasDictionarySupport() { - field.logicalTypeAnnotation instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation || expected == String + MatrixParquetReader.isTextLogicalType(field.logicalTypeAnnotation) || expected == String } @Override void setDictionary(Dictionary source) { dictionary = new Object[source.maxId + 1] if (logicalSink != null) { logicalDictionary = new Object[source.maxId + 1] } - (0..source.maxId).each { int id -> + for (int id = 0; id <= source.maxId; id++) { Binary value = source.decodeToBinary(id) dictionary[id] = MatrixParquetReader.decodePrimitive(value, field, expected, binaryAsString) if (logicalSink != null) { logicalDictionary[id] = logicalValue(value) } diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DatasetBatches.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DatasetBatches.groovy index 1d81f1df7..0ce753e36 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DatasetBatches.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/DatasetBatches.groovy @@ -45,9 +45,18 @@ final class DatasetBatches implements MatrixBatchIterator { remaining = options.limit == null ? Long.MAX_VALUE : options.limit ParquetReadOptions emptyOptions = ParquetReadOptions.fromMap(options.toMap()) emptyOptions.rowGroups = [] + emptyOptions.sourceFileColumn = null new ParquetBatches(new LocalInputFile(files[0].toPath()), emptyOptions, files[0].name, footers[0]).withCloseable { cursor -> template = cursor.emptyMatrix() } + if (!footers.every { it.fileMetaData.keyValueMetaData[MatrixParquetReader.METADATA_INDEX_COLUMNS] == footers[0].fileMetaData.keyValueMetaData[MatrixParquetReader.METADATA_INDEX_COLUMNS] }) { + template = Matrix.builder(template.matrixName).columnNames(template.columnNames()).types(template.types()).build() + } if (options.sourceFileColumn != null) { template.addColumn(options.sourceFileColumn, String, []) } - executor = Executors.newFixedThreadPool(Math.min(options.parallelism, Math.max(1, work.size()))) + executor = Executors.newFixedThreadPool(Math.min(options.parallelism, Math.max(1, work.size())), + { Runnable action -> + Thread worker = new Thread(action, 'matrix-parquet-dataset') + worker.daemon = true + worker + } as java.util.concurrent.ThreadFactory) try { if (remaining > 0) { (0.. 0) { scheduleBatch(current) } return true @@ -147,6 +160,17 @@ final class DatasetBatches implements MatrixBatchIterator { } } + private Matrix withDatasetIndex(Matrix batch) { + if (template.hasIndex()) { + if (!batch.hasIndex()) { batch.createIndex(template.indexedColumns() as String[]) } + } else if (batch.hasIndex()) { + List columns = batch.columnNames().collect { (List) batch.column(it) } + return Matrix.builder(batch.matrixName).columnNames(batch.columnNames()).types(batch.types()) + .columns(columns).build() + } + batch + } + @Override Matrix next() { if (!hasNext()) { throw new NoSuchElementException() } diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpRangeInputFile.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpRangeInputFile.groovy index e43455a4a..c93341cf0 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpRangeInputFile.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpRangeInputFile.groovy @@ -11,7 +11,10 @@ final class HttpRangeInputFile implements InputFile { private static final String ETAG = 'ETag' private static final String LAST_MODIFIED = 'Last-Modified' + private static final int MAX_REQUEST_BYTES = 8388608 + private static final int PREFETCH_BYTES = 65536 private static final int EOF = -1 + private volatile URL resolvedTarget private final URL source private final ParquetReadOptions options private final int cacheBytes @@ -24,10 +27,11 @@ final class HttpRangeInputFile implements InputFile { if (!(source.protocol in ['http', 'https'])) { throw new IOException('Range reads require HTTP(S)') } this.source = source this.options = options - cacheBytes = Math.min(options.rangeCacheBytes, 65536) + cacheBytes = options.rangeCacheBytes def response = request(0L, 0L) def connection = response.connection + boolean complete = false try { String range = connection.getHeaderField(CONTENT_RANGE) def match = range =~ /bytes 0-0\/(\d+)/ @@ -39,14 +43,21 @@ final class HttpRangeInputFile implements InputFile { connection.inputStream.withCloseable { stream -> if (stream.read() < 0 || stream.read() != EOF) { throw new IOException('Invalid range payload length') } } - } finally { connection.disconnect() } + resolvedTarget = connection.URL + complete = true + } finally { if (!complete) { connection.disconnect() } } } private Map request(long start, long end) { Map headers = [Range: "bytes=$start-$end".toString()] if (etag != null) { headers['If-Match'] = etag } else if (modified != null) { headers['If-Unmodified-Since'] = modified } - HttpURLConnection connection = HttpTransport.open(source, options, headers) + URL target = resolvedTarget ?: source + HttpURLConnection connection = HttpTransport.open(target, options, headers) + if (target.toExternalForm() != source.toExternalForm() && connection.responseCode in [401, 403]) { + connection.disconnect() + connection = HttpTransport.open(source, options, headers) + } if (connection.responseCode != 206 || !(connection.getHeaderField('Content-Encoding') in [null, 'identity'])) { int status = connection.responseCode connection.disconnect() @@ -58,6 +69,7 @@ final class HttpRangeInputFile implements InputFile { private byte[] fetch(long start, int count) { long end = start + count - 1 HttpURLConnection connection = request(start, end).connection + boolean complete = false try { if (connection.getHeaderField(CONTENT_RANGE) != "bytes $start-$end/$length".toString() || (etag != null && connection.getHeaderField(ETAG) != etag) || @@ -67,9 +79,11 @@ final class HttpRangeInputFile implements InputFile { connection.inputStream.withCloseable { stream -> byte[] bytes = stream.readNBytes(count) if (bytes.length != count || stream.read() != EOF) { throw new EOFException('Truncated/oversized HTTP range') } + resolvedTarget = connection.URL + complete = true bytes } - } finally { connection.disconnect() } + } finally { if (!complete) { connection.disconnect() } } } @Override @@ -101,12 +115,17 @@ final class HttpRangeInputFile implements InputFile { if (closed) { throw new IOException('Stream closed') } if (count == 0) { return 0 } if (position >= length) { return EOF } + byte[] available = cache + long start = cacheStart if (position < cacheStart || position >= cacheStart + cache.length) { - cacheStart = position - cache = fetch(position, (int) Math.min(cacheBytes, length - position)) + start = position + int requested = Math.max(Math.min(count, MAX_REQUEST_BYTES), Math.min(cacheBytes, PREFETCH_BYTES)) + available = fetch(position, (int) Math.min(requested, length - position)) + cacheStart = start + cache = available.length <= cacheBytes ? available : new byte[0] } - int copied = (int) Math.min(count, cacheStart + cache.length - position) - System.arraycopy(cache, (int) (position - cacheStart), target, offset, copied) + int copied = (int) Math.min(count, start + available.length - position) + System.arraycopy(available, (int) (position - start), target, offset, copied) position += copied copied } @@ -123,7 +142,7 @@ final class HttpRangeInputFile implements InputFile { } @Override int read(ByteBuffer target) { - byte[] bytes = new byte[Math.min(target.remaining(), cacheBytes)] + byte[] bytes = new byte[Math.min(target.remaining(), MAX_REQUEST_BYTES)] int result = read(bytes, 0, bytes.length) if (result > 0) { target.put(bytes, 0, result) } result diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HubUrl.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HubUrl.groovy index 7b300c546..1a8934fff 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HubUrl.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HubUrl.groovy @@ -8,6 +8,7 @@ final class HubUrl { private static final String SLASH = '/' private static final String ERR_FILE_PATH = 'A dataset file path is required' private static final String PLUS = '+' + private static final String PARENT = '..' private static final int PARTS = 3 private static final String PREFIX = 'hf://datasets/' @@ -46,8 +47,8 @@ final class HubUrl { if (!(repo ==~ /[^\/@]+\/[^\/@]+/) || revision.empty || path.empty) { throw new IllegalArgumentException('Invalid dataset repository, revision or file path') } - if (repo.contains('--') || repo.contains('..') || repo.endsWith('.git') || - repo.split(SLASH).any { !(it ==~ /[\w](?:[\w.-]*[\w])?/) } || path.startsWith(SLASH) || path.contains('//')) { + if (repo.contains('--') || repo.contains(PARENT) || repo.endsWith('.git') || + repo.split(SLASH).any { !(it ==~ /[\w](?:[\w.-]*[\w])?/) } || path.startsWith(SLASH) || path.contains('//') || hasDotSegments(path) || hasDotSegments(revision)) { throw new IllegalArgumentException('Invalid repository id or empty path segment') } String encodedRevision = SPECIAL.matcher(revision).matches() ? revision : encode(revision) @@ -55,6 +56,10 @@ final class HubUrl { new URI("https://huggingface.co/datasets/$repo/resolve/$encodedRevision/$encodedPath").toURL() } + private static boolean hasDotSegments(String value) { + value.split(SLASH).any { it in ['.', PARENT] } + } + private static String encode(String value) { URLEncoder.encode(value, StandardCharsets.UTF_8).replace(PLUS, '%20').replace('%7E', '~').replace('*', '%2A') } diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetReader.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetReader.groovy index 92a9251a7..e8b0e8f9e 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetReader.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetReader.groovy @@ -78,7 +78,7 @@ import java.time.ZoneOffset * *

Limitations

*
    - *
  • The entire file is loaded into memory (no streaming support)
    • + *
  • read() collects the result in memory; closeable batches bound accepted rows
    • *
* * @see MatrixParquetWriter @@ -106,7 +106,7 @@ class MatrixParquetReader { /** Thread-local storage for timezone used during read operations */ private static final ThreadLocal ZONE_ID_HOLDER = new ThreadLocal<>() - /** Cache for Class.forName() results to avoid repeated reflection calls */ + /** Binary text override used by the retained Group benchmark helper */ private static final ThreadLocal BINARY_AS_STRING = new ThreadLocal<>() private static final String ERR_PATH_NULL = 'Path cannot be null' @@ -233,7 +233,7 @@ class MatrixParquetReader { MatrixBatchIterator batches(File file) { if (file?.isDirectory()) { return batches(directoryFiles(file)) } validateFile(file) - new ParquetBatches(new org.apache.parquet.io.LocalInputFile(file.toPath()), options, defaultMatrixName(file)) + new ParquetBatches(new org.apache.parquet.io.LocalInputFile(file.toPath()), options, defaultMatrixName(file), null, file.canonicalPath) } @@ -279,7 +279,7 @@ class MatrixParquetReader { Matrix read(File file) { if (file?.isDirectory()) { return read(directoryFiles(file)) } validateFile(file) - readInput(new org.apache.parquet.io.LocalInputFile(file.toPath()), options, defaultMatrixName(file)) + readInput(new org.apache.parquet.io.LocalInputFile(file.toPath()), options, defaultMatrixName(file), file.canonicalPath) } /** @@ -386,7 +386,7 @@ class MatrixParquetReader { * Validates the file parameter before reading. * * @param file the file to validate - * @throws IllegalArgumentException if file is null, does not exist, or is a directory + * @throws IllegalArgumentException if file is null or does not exist */ private static void validateFile(File file) { if (file == null) { @@ -413,10 +413,10 @@ class MatrixParquetReader { * @param file the Parquet file to read * @param matrixName the name to apply to the resulting matrix * @return a matrix populated with the file contents - * @throws IllegalArgumentException if file is null, does not exist, or is a directory + * @throws IllegalArgumentException if file is null or does not exist */ static Matrix read(File file, String matrixName) { - read(file, null, false, null).withMatrixName(matrixName) + builder().matrixName(matrixName).read(file) } /** @@ -424,7 +424,7 @@ class MatrixParquetReader { * * @param path the Path to read * @return a matrix populated with the file contents - * @throws IllegalArgumentException if path is null, does not exist, or is a directory + * @throws IllegalArgumentException if path is null or does not exist */ static Matrix read(java.nio.file.Path path) { if (path == null) { @@ -439,7 +439,7 @@ class MatrixParquetReader { * @param path the Path to read * @param matrixName the name to apply to the resulting matrix * @return a matrix populated with the file contents - * @throws IllegalArgumentException if path is null, does not exist, or is a directory + * @throws IllegalArgumentException if path is null or does not exist */ static Matrix read(java.nio.file.Path path, String matrixName) { if (path == null) { @@ -453,7 +453,7 @@ class MatrixParquetReader { * * @param filePath the file path to read * @return a matrix populated with the file contents - * @throws IllegalArgumentException if filePath is null, does not exist, or is a directory + * @throws IllegalArgumentException if filePath is null or does not exist */ static Matrix readFile(String filePath) { if (filePath == null) { @@ -468,7 +468,7 @@ class MatrixParquetReader { * @param filePath the file path to read * @param matrixName the name to apply to the resulting matrix * @return a matrix populated with the file contents - * @throws IllegalArgumentException if filePath is null, does not exist, or is a directory + * @throws IllegalArgumentException if filePath is null or does not exist */ static Matrix readFile(String filePath, String matrixName) { if (filePath == null) { @@ -480,7 +480,7 @@ class MatrixParquetReader { /** * Read Parquet content from a byte array into a {@link Matrix}. * - *

Parquet requires a seekable input, so the content is copied to a temporary file.

+ *

The byte array is read directly through a seekable in-memory input.

* * @param content byte array containing Parquet data * @return a matrix populated with the byte array contents @@ -496,7 +496,7 @@ class MatrixParquetReader { /** * Read Parquet content from a byte array into a {@link Matrix} using the supplied name. * - *

Parquet requires a seekable input, so the content is copied to a temporary file.

+ *

The byte array is read directly through a seekable in-memory input.

* * @param content byte array containing Parquet data * @param matrixName the name to apply to the resulting matrix @@ -513,7 +513,7 @@ class MatrixParquetReader { /** * Read Parquet content from a byte array into a {@link Matrix} with a specific timezone. * - *

Parquet requires a seekable input, so the content is copied to a temporary file.

+ *

The byte array is read directly through a seekable in-memory input.

* * @param content byte array containing Parquet data * @param zoneId the timezone to use for converting UTC timestamps to LocalDateTime values @@ -530,7 +530,7 @@ class MatrixParquetReader { /** * Read Parquet content from a byte array into a {@link Matrix} with a specific timezone and name. * - *

Parquet requires a seekable input, so the content is copied to a temporary file.

+ *

The byte array is read directly through a seekable in-memory input.

* * @param content byte array containing Parquet data * @param matrixName the name to apply to the resulting matrix @@ -583,9 +583,7 @@ class MatrixParquetReader { if (url == null) { throw new IllegalArgumentException(ERR_URL_NULL) } - url.openStream().withCloseable { InputStream is -> - readFromInputStream(is, null, null) - } + builder().read(url) } /** @@ -611,13 +609,13 @@ class MatrixParquetReader { * @param file the Parquet file to read * @param zoneId the timezone to use for converting UTC timestamps to LocalDateTime values * @return a matrix populated with the file contents - * @throws IllegalArgumentException if file is null, does not exist, is a directory, or zoneId is null + * @throws IllegalArgumentException if file is null, does not exist, or zoneId is null */ static Matrix read(File file, ZoneId zoneId) { if (zoneId == null) { throw new IllegalArgumentException(ERR_ZONE_ID_NULL) } - read(file, file == null ? null : defaultMatrixName(file), false, zoneId) + builder().zoneId(zoneId).read(file) } /** @@ -626,7 +624,7 @@ class MatrixParquetReader { * @param path the Path to read * @param zoneId the timezone to use for converting UTC timestamps to LocalDateTime values * @return a matrix populated with the file contents - * @throws IllegalArgumentException if path is null, does not exist, or is a directory, or zoneId is null + * @throws IllegalArgumentException if path is null, does not exist, or zoneId is null */ static Matrix read(java.nio.file.Path path, ZoneId zoneId) { if (path == null) { @@ -682,9 +680,7 @@ class MatrixParquetReader { if (zoneId == null) { throw new IllegalArgumentException(ERR_ZONE_ID_NULL) } - url.openStream().withCloseable { InputStream is -> - readFromInputStream(is, null, zoneId) - } + builder().zoneId(zoneId).read(url) } /** @@ -699,7 +695,8 @@ class MatrixParquetReader { if (urlString == null) { throw new IllegalArgumentException(ERR_URL_STRING_NULL) } - read(new URI(urlString).toURL(), zoneId) + if (zoneId == null) { throw new IllegalArgumentException(ERR_ZONE_ID_NULL) } + builder().zoneId(zoneId).readUrl(urlString) } /** @@ -709,13 +706,13 @@ class MatrixParquetReader { * @param matrixName the name to apply to the resulting matrix * @param zoneId the timezone to use for converting UTC timestamps to LocalDateTime values * @return a matrix populated with the file contents - * @throws IllegalArgumentException if file is null, does not exist, is a directory, or zoneId is null + * @throws IllegalArgumentException if file is null, does not exist, or zoneId is null */ static Matrix read(File file, String matrixName, ZoneId zoneId) { if (zoneId == null) { throw new IllegalArgumentException(ERR_ZONE_ID_NULL) } - read(file, null, false, zoneId).withMatrixName(matrixName) + builder().matrixName(matrixName).zoneId(zoneId).read(file) } /** @@ -784,9 +781,7 @@ class MatrixParquetReader { if (zoneId == null) { throw new IllegalArgumentException(ERR_ZONE_ID_NULL) } - url.openStream().withCloseable { InputStream is -> - readFromInputStream(is, matrixName, zoneId) - } + builder().matrixName(matrixName).zoneId(zoneId).read(url) } /** @@ -802,7 +797,8 @@ class MatrixParquetReader { if (urlString == null) { throw new IllegalArgumentException(ERR_URL_STRING_NULL) } - read(new URI(urlString).toURL(), matrixName, zoneId) + if (zoneId == null) { throw new IllegalArgumentException(ERR_ZONE_ID_NULL) } + builder().matrixName(matrixName).zoneId(zoneId).readUrl(urlString) } /** @@ -810,28 +806,11 @@ class MatrixParquetReader { * * @param file the Parquet file to read * @return a matrix populated with the file contents - * @throws IllegalArgumentException if file is null, does not exist, or is a directory + * @throws IllegalArgumentException if file is null or does not exist */ static Matrix read(File file) { if (file?.isDirectory()) { return builder().read(file) } - read(file, file == null ? null : defaultMatrixName(file), false, null) - } - - private static Matrix read(File file, String fallbackMatrixName, boolean preferSchemaName, ZoneId zoneId) { - if (zoneId == null) { - return read(file, fallbackMatrixName, preferSchemaName) - } - try { - ZONE_ID_HOLDER.set(zoneId) - return read(file, fallbackMatrixName, preferSchemaName) - } finally { - ZONE_ID_HOLDER.remove() - } - } - - private static Matrix read(File file, String fallbackMatrixName, boolean preferSchemaName) { - validateFile(file) - readInput(new org.apache.parquet.io.LocalInputFile(file.toPath()), new ParquetReadOptions(), preferSchemaName ? null : fallbackMatrixName) + builder().read(file) } /** Reads a list of shards in the supplied order, with strict schema compatibility. */ @@ -866,7 +845,9 @@ class MatrixParquetReader { Matrix batch = iterator.next() batch.columnNames().eachWithIndex { String name, int index -> columns[index].addAll(batch.column(name)) } } - Matrix.builder(prototype.matrixName).columnNames(prototype.columnNames()).types(prototype.types()).columns(columns).build() + Matrix result = Matrix.builder(prototype.matrixName).columnNames(prototype.columnNames()).types(prototype.types()).columns(columns).build() + if (prototype.hasIndex()) { result.createIndex(prototype.indexedColumns() as String[]) } + result } @PackageScope @@ -879,21 +860,13 @@ class MatrixParquetReader { } @PackageScope - static Matrix readInput(org.apache.parquet.io.InputFile input, ParquetReadOptions options, String name) { + static Matrix readInput(org.apache.parquet.io.InputFile input, ParquetReadOptions options, String name, String sourcePath = null) { log.debug("Reading Parquet input: $name") ZoneId previous = ZONE_ID_HOLDER.get() if (options.zoneId != null) { ZONE_ID_HOLDER.set(options.zoneId) } try { - new ParquetBatches(input, options, name).withCloseable { batches -> - Matrix empty = batches.emptyMatrix() - List columns = empty.columnNames().collect { [] } - while (batches.hasNext()) { - Matrix batch = batches.next() - batch.columnNames().eachWithIndex { String column, int index -> columns[index].addAll(batch.column(column)) } - } - Matrix result = Matrix.builder(empty.matrixName).columnNames(empty.columnNames()).types(empty.types()).columns(columns).build() - if (empty.hasIndex()) { result.createIndex(empty.indexedColumns() as String[]) } - result + new ParquetBatches(input, options, name, null, sourcePath).withCloseable { batches -> + collectBatches(batches) } } finally { if (previous == null) { ZONE_ID_HOLDER.remove() } else { ZONE_ID_HOLDER.set(previous) } @@ -1023,28 +996,9 @@ class MatrixParquetReader { } private static Matrix readFromInputStream(InputStream is, String matrixName, ZoneId zoneId) { - if (is == null) { - throw new IllegalArgumentException('InputStream cannot be null') - } - java.nio.file.Path tempFile = Files.createTempFile('matrix-parquet-', EXTENSION) - try { - Files.copy(is, tempFile, StandardCopyOption.REPLACE_EXISTING) - File file = tempFile.toFile() - Matrix result = read(file, defaultMatrixName(file), true, zoneId) - if (matrixName != null && !matrixName.trim().isEmpty()) { - result = result.withMatrixName(matrixName) - } - result - } finally { - try { - Files.deleteIfExists(tempFile) - } catch (IOException ignored) { - } - } + builder().matrixName(matrixName).zoneId(zoneId).read(is) } - - private static Class getJavaType(PrimitiveType.PrimitiveTypeName typeName) { switch (typeName) { case PrimitiveTypeName.INT32 -> Integer @@ -1057,6 +1011,13 @@ class MatrixParquetReader { } } + @PackageScope + static boolean isTextLogicalType(LogicalTypeAnnotation logical) { + logical instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation || + logical instanceof LogicalTypeAnnotation.JsonLogicalTypeAnnotation || + logical instanceof LogicalTypeAnnotation.EnumLogicalTypeAnnotation + } + @PackageScope static List extractFieldTypes(GroupType schema) { schema.fields.collect { field -> @@ -1079,7 +1040,7 @@ class MatrixParquetReader { BigDecimal } else if (logical instanceof LogicalTypeAnnotation.TimeLogicalTypeAnnotation) { Time - } else if (logical instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation) { + } else if (isTextLogicalType(logical)) { String } else if (logical instanceof LogicalTypeAnnotation.UUIDLogicalTypeAnnotation) { UUID @@ -1166,7 +1127,7 @@ class MatrixParquetReader { } if (raw instanceof org.apache.parquet.io.api.Binary) { if (expected == BigDecimal) { return new BigDecimal(raw.toStringUsingUTF8()) } - if (logical instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation || expected == String || + if (isTextLogicalType(logical) || expected == String || (logical == null && field.primitiveTypeName == PrimitiveTypeName.BINARY && binaryAsString && expected != byte[])) { return raw.toStringUsingUTF8() } diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy index 8f0fc5412..53afd7c13 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy @@ -339,7 +339,7 @@ class MatrixParquetWriter { */ static File write(Matrix matrix, File fileOrDir, boolean inferPrecisionAndScale = true) { validateInput(matrix, fileOrDir) - MessageType schema + WriteSchema schema if (inferPrecisionAndScale) { schema = buildSchema(matrix,true) } else { @@ -449,7 +449,7 @@ class MatrixParquetWriter { } options.validate() File file = determineTargetFile(matrix, fileOrDir) - MessageType schema = createSchema(matrix, options) + WriteSchema schema = createSchema(matrix, options) if (options.zoneId != null) { return ParquetValueWriter.withZoneId(options.zoneId) { writeInternal(matrix, file, schema, options.compressionCodec, options.roundingMode, options) @@ -469,7 +469,7 @@ class MatrixParquetWriter { */ static byte[] writeBytes(Matrix matrix, boolean inferPrecisionAndScale = true) { validateInput(matrix) - MessageType schema = buildSchema(matrix, inferPrecisionAndScale) + WriteSchema schema = buildSchema(matrix, inferPrecisionAndScale) return writeBytesInternal(matrix, schema) } @@ -498,7 +498,7 @@ class MatrixParquetWriter { */ static byte[] writeBytes(Matrix matrix, Map decimalMeta) { validateInput(matrix) - MessageType schema = buildSchema(matrix, ParquetWriteOptions.normalizeDecimalMeta(decimalMeta ?: [:])) + WriteSchema schema = buildSchema(matrix, ParquetWriteOptions.normalizeDecimalMeta(decimalMeta ?: [:])) return writeBytesInternal(matrix, schema) } @@ -515,7 +515,7 @@ class MatrixParquetWriter { throw new IllegalArgumentException(ERR_OPTIONS_NULL) } options.validate() - MessageType schema = createSchema(matrix, options) + WriteSchema schema = createSchema(matrix, options) if (options.zoneId != null) { return ParquetValueWriter.withZoneId(options.zoneId) { writeBytesInternal(matrix, schema, options.compressionCodec, options.roundingMode, options) @@ -549,7 +549,7 @@ class MatrixParquetWriter { normalized.toLowerCase(Locale.ROOT).endsWith('.parquet') ? normalized : "${normalized}.parquet" } - private static MessageType createSchema(Matrix matrix, ParquetWriteOptions options) { + private static WriteSchema createSchema(Matrix matrix, ParquetWriteOptions options) { buildSchema(matrix, options.decimalMeta, new DecimalSchemaPolicy(options.decimalMeta, options.inferPrecisionAndScale, options.precision, options.scale)) } @@ -559,21 +559,29 @@ class MatrixParquetWriter { * to a Parquet file, given an already constructed schema. * @param matrix the matrix to write * @param file the target file - * @param schema the Parquet MessageType schema + * @param resolved the validated physical schema and column policies * @return the target file */ - private static File writeInternal(Matrix matrix, File file, MessageType schema, + private static File writeInternal(Matrix matrix, File file, WriteSchema resolved, CompressionCodecName compressionCodec = CompressionCodecName.SNAPPY, RoundingMode roundingMode = RoundingMode.UNNECESSARY, ParquetWriteOptions options = new ParquetWriteOptions()) { - def conf = new org.apache.parquet.conf.PlainParquetConfiguration() - def writerBuilder = ExampleParquetWriter.builder(new org.apache.parquet.io.LocalOutputFile(file.toPath())) - .withConf(conf) - .withWriteMode(ParquetFileWriter.Mode.OVERWRITE) - .withType(schema) - .withCompressionCodec(compressionCodec) - .withExtraMetaData(buildExtraMeta(matrix)) - def writer = configureWriter(writerBuilder, schema, options).build() - writer.withCloseable { parquetWriter -> writeRows(parquetWriter, matrix, schema, roundingMode) } - return file + MessageType schema = resolved.schema + java.nio.file.Path target = file.toPath().toAbsolutePath() + java.nio.file.Path temporary = java.nio.file.Files.createTempFile(target.parent, '.matrix-parquet-', '.tmp') + try { + if (java.nio.file.Files.exists(target) && java.nio.file.Files.getFileStore(target).supportsFileAttributeView('posix')) { + java.nio.file.Files.setPosixFilePermissions(temporary, java.nio.file.Files.getPosixFilePermissions(target)) + } + def conf = new org.apache.parquet.conf.PlainParquetConfiguration() + def writerBuilder = ExampleParquetWriter.builder(new org.apache.parquet.io.LocalOutputFile(temporary)) + .withConf(conf).withWriteMode(ParquetFileWriter.Mode.OVERWRITE).withType(schema) + .withCompressionCodec(compressionCodec).withExtraMetaData(buildExtraMeta(matrix, resolved)) + def writer = configureWriter(writerBuilder, schema, options).build() + writer.withCloseable { parquetWriter -> writeRows(parquetWriter, matrix, schema, roundingMode) } + // Fail safely if the filesystem cannot atomically replace the destination. + java.nio.file.Files.move(temporary, target, java.nio.file.StandardCopyOption.ATOMIC_MOVE, + java.nio.file.StandardCopyOption.REPLACE_EXISTING) + file + } finally { java.nio.file.Files.deleteIfExists(temporary) } } private static final String FORMAT_VERSION = '2' @@ -596,9 +604,9 @@ class MatrixParquetWriter { builder } - private static Map buildExtraMeta(Matrix matrix) { + private static Map buildExtraMeta(Matrix matrix, WriteSchema resolved) { Map extraMeta = ['matrix.parquet.version': FORMAT_VERSION] - extraMeta.put(METADATA_COLUMN_TYPES, matrix.columnNames().collect { String col -> ColumnTypePolicy.resolve(col, matrix.column(col), matrix.type(col)).metadata.name }.join(COMMA)) + extraMeta.put(METADATA_COLUMN_TYPES, matrix.columnNames().collect { String col -> resolved.types[col].metadata.name }.join(COMMA)) if (matrix.hasIndex()) { extraMeta.put(METADATA_INDEX_COLUMNS, toJsonStringArray(matrix.indexedColumns())) } @@ -638,18 +646,19 @@ class MatrixParquetWriter { * constraints (Integer.MAX_VALUE).

* * @param matrix the matrix to write - * @param schema the Parquet MessageType schema + * @param resolved the validated physical schema and column policies * @return byte array containing Parquet data */ - private static byte[] writeBytesInternal(Matrix matrix, MessageType schema, + private static byte[] writeBytesInternal(Matrix matrix, WriteSchema resolved, CompressionCodecName compressionCodec = CompressionCodecName.SNAPPY, RoundingMode roundingMode = RoundingMode.UNNECESSARY, ParquetWriteOptions options = new ParquetWriteOptions()) { + MessageType schema = resolved.schema def conf = new org.apache.parquet.conf.PlainParquetConfiguration() InMemoryOutputFile outputFile = new InMemoryOutputFile() def writerBuilder = ExampleParquetWriter.builder(outputFile) .withConf(conf) .withType(schema) .withCompressionCodec(compressionCodec) - .withExtraMetaData(buildExtraMeta(matrix)) + .withExtraMetaData(buildExtraMeta(matrix, resolved)) def writer = configureWriter(writerBuilder, schema, options).build() writer.withCloseable { parquetWriter -> writeRows(parquetWriter, matrix, schema, roundingMode) } return outputFile.getBytes() @@ -693,22 +702,25 @@ class MatrixParquetWriter { * * @param matrix the matrix * @param explicitDecimalMeta map of column name to [precision, scale] array for BigDecimal columns. - * @return the Parquet MessageType schema. + * @return validated physical schema and metadata policies. */ - private static MessageType buildSchema(Matrix matrix, Map explicitDecimalMeta, DecimalSchemaPolicy decimals = null) { - Map decimalMeta = withBigIntegerPrecision(matrix, explicitDecimalMeta) + private static WriteSchema buildSchema(Matrix matrix, Map explicitDecimalMeta, DecimalSchemaPolicy decimals = null) { + Map types = matrix.columnNames().collectEntries { String name -> + [(name): ColumnTypePolicy.resolve(name, matrix.column(name), matrix.type(name))] + } + Map decimalMeta = withBigIntegerPrecision(matrix, explicitDecimalMeta, types) DecimalSchemaPolicy policy = decimals ?: new DecimalSchemaPolicy(explicitDecimalMeta ?: [:]) def builder = Types.buildMessage() matrix.columnNames().each { col -> def type = matrix.type(col) def meta = decimalMeta?.get(col) log.debug("Building schema for column '$col' of type $type with decimal meta: $meta") - builder.addField(buildParquetType(col, matrix.column(col), type, meta, false, policy.child(col))) + builder.addField(buildParquetType(col, matrix.column(col), type, meta, false, policy.child(col), types[col])) } Set unknown = new HashSet<>(policy.settings.keySet()) unknown.removeAll(policy.used) if (!unknown.empty) { throw new IllegalArgumentException("Unknown/non-decimal metadata paths: $unknown") } - return builder.named(matrix.matrixName ?: 'MatrixSchema') + new WriteSchema(builder.named(matrix.matrixName ?: 'MatrixSchema'), types) } /** @@ -716,10 +728,10 @@ class MatrixParquetWriter { * Explicit caller-supplied metadata wins after validating its scale is 0 * and that its precision is large enough to hold every value in the column. */ - private static Map withBigIntegerPrecision(Matrix matrix, Map explicitDecimalMeta) { + private static Map withBigIntegerPrecision(Matrix matrix, Map explicitDecimalMeta, Map types) { Map merged = [:] + (explicitDecimalMeta ?: [:]) matrix.columnNames().each { col -> - if (ColumnTypePolicy.resolve(col, matrix.column(col), matrix.type(col)).encoding != BigInteger) { + if (types[col].encoding != BigInteger) { return } int[] existing = merged[col] @@ -728,35 +740,19 @@ class MatrixParquetWriter { throw new IllegalArgumentException( "decimalMeta['$col'] scale must be 0 for BigInteger columns but was ${existing[1]}") } - int requiredPrecision = inferBigIntegerPrecision(matrix, col) + int requiredPrecision = inferBigIntegerPrecision(matrix.column(col)) if (existing[0] < requiredPrecision) { throw new IllegalArgumentException( "decimalMeta['$col'] precision (${existing[0]}) is too small to hold the data in this " + "column (requires at least $requiredPrecision digits)") } } else { - merged[col] = [inferBigIntegerPrecision(matrix, col), 0] as int[] + merged[col] = [inferBigIntegerPrecision(matrix.column(col)), 0] as int[] } } merged } - /** - * Infers the minimum decimal precision needed to represent every BigInteger - * value in the given column as Parquet DECIMAL(precision, 0). - */ - private static int inferBigIntegerPrecision(Matrix matrix, String col) { - int maxDigits = 1 - (0.. - def value = matrix[i, col] - if (value instanceof BigInteger) { - int digits = value.abs().toString().length() - maxDigits = Math.max(maxDigits, digits) - } - } - maxDigits - } - private static int inferBigIntegerPrecision(List values) { int maxDigits = 1 values.each { value -> @@ -774,14 +770,10 @@ class MatrixParquetWriter { * @param matrix the matrix * @param inferPrecisionAndScale if true, infers decimal precision and scale. If false, BigDecimal fields require * explicit precision/scale settings. - * @return the Parquet MessageType schema. + * @return validated physical schema and metadata policies. */ - private static MessageType buildSchema(Matrix matrix, boolean inferPrecisionAndScale = false) { - Map decimalMeta = null - if (inferPrecisionAndScale) { - decimalMeta = inferDecimalPrecisionAndScale(matrix) - } - return buildSchema(matrix, decimalMeta, new DecimalSchemaPolicy(decimalMeta ?: [:], inferPrecisionAndScale)) + private static WriteSchema buildSchema(Matrix matrix, boolean inferPrecisionAndScale = false) { + buildSchema(matrix, [:], new DecimalSchemaPolicy([:], inferPrecisionAndScale)) } /** @@ -793,8 +785,8 @@ class MatrixParquetWriter { * @param decimalMeta optional decimal precision/scale metadata for {@link BigDecimal} columns * @return a Parquet {@link Type} describing the column */ - private static Type buildParquetType(String name, List values, Class clazz, int[] decimalMeta = null, boolean preferStructForMaps = false, DecimalSchemaPolicy decimals = new DecimalSchemaPolicy([:])) { - ColumnTypePolicy resolved = ColumnTypePolicy.resolve(decimals.path ?: name, values, clazz) + private static Type buildParquetType(String name, List values, Class clazz, int[] decimalMeta = null, boolean preferStructForMaps = false, DecimalSchemaPolicy decimals = new DecimalSchemaPolicy([:]), ColumnTypePolicy resolved = null) { + resolved = resolved ?: ColumnTypePolicy.resolve(decimals.path ?: name, values, clazz) if (resolved.encoding == List) { List nestedValues = extractListElements(values) Type elementType = buildParquetType(FIELD_ELEMENT, nestedValues, Object, null, true, decimals.child(FIELD_ELEMENT)) @@ -907,8 +899,7 @@ class MatrixParquetWriter { if (maps.any { it.containsKey(null) }) { throw new IllegalArgumentException("Map field '$name' cannot contain null keys") } List keySamples = maps.collectMany { it?.keySet() ?: [] } as List - Object keySample = findFirstNonNull(keySamples) - Class keyClass = inferClass(keySamples, keySample?.class ?: String) + Class keyClass = ColumnTypePolicy.resolve(decimals.child(FIELD_KEY).path, keySamples, Object).encoding int[] keyDecimalMeta = keyClass == BigInteger ? [inferBigIntegerPrecision(keySamples), 0] as int[] : keyClass == BigDecimal ? decimals.child(FIELD_KEY).resolve(keySamples, null) : null PrimitiveType keyType = buildPrimitiveType(FIELD_KEY, keyClass ?: String, keyDecimalMeta, true) mapBuilder.key(keyType) @@ -997,10 +988,6 @@ class MatrixParquetWriter { return elements } - private static Class inferClass(List values, Class declaredType) { - ColumnTypePolicy.resolve('nested field', values, declaredType).encoding - } - private static Object findFirstNonNull(List values) { for (def val : values) { if (val != null) { @@ -1010,56 +997,18 @@ class MatrixParquetWriter { return null } - /** - * Infers the required maximum precision and scale for all BigDecimal columns in the matrix. - * It performs the calculation in a single pass: - * 1. It tracks the running maximum scale (S). - * 2. It tracks the running maximum number of digits to the LEFT of the decimal (L). - * 3. The final precision P = L + S. - * This guarantees that the resulting schema is large enough to contain all values. - * @param matrix the matrix - * @return a Map of column name to [precision, scale] array. - */ - private static Map inferDecimalPrecisionAndScale(Matrix matrix) { - Map result = [:] // columnName -> [precision, scale] - - matrix.columnNames().each { col -> - if (ColumnTypePolicy.resolve(col, matrix.column(col), matrix.type(col)).encoding == BigDecimal) { - int maxScale = 0 - int maxLeftDigits = 0 // Max digits to the left of the decimal - - (0.. - def value = matrix[i, col] - if (value instanceof BigDecimal) { - int valScale = value.scale() - maxScale = Math.max(maxScale, valScale) - - // Calculate digits to the left of the decimal - // precision() - scale() works for most numbers (e.g., 123.45 -> p=5, s=2 -> 3) - // but fails for numbers < 1 (e.g., 0.45 -> p=2, s=2 -> 0) - // We must ensure at least one digit (for the '0') - int leftDigits = Math.max(1, value.precision() - value.scale()) - maxLeftDigits = Math.max(maxLeftDigits, leftDigits) - } - } - - // Total precision = max digits left + max digits right (max scale) - int maxPrecision = maxLeftDigits + maxScale + private static int minBytesForPrecision(int precision) { + int bitsNeeded = BigInteger.TEN.pow(precision).subtract(BigInteger.ONE).bitLength() + 1 + Math.ceil(bitsNeeded / 8d) as int + } - // Ensure precision is at least 1 (e.g., for a column of all 0s) - maxPrecision = Math.max(1, maxPrecision) + private static final class WriteSchema { + final MessageType schema + final Map types - result[col] = [maxPrecision, maxScale] as int[] - } + WriteSchema(MessageType schema, Map types) { + this.schema = schema + this.types = types } - log.debug("Inferred decimal precision/scale: $result") - return result - } - - private static int minBytesForPrecision(int precision) { - // According to the Parquet spec: enough bits to hold [-(10^precision - 1), 10^precision - 1] - // in two's complement, i.e. the bit length of the largest magnitude plus a sign bit. - int bitsNeeded = BigInteger.TEN.pow(precision).subtract(BigInteger.ONE).bitLength() + 1 - return Math.ceil(bitsNeeded / 8d) as int } } diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetBatches.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetBatches.groovy index ce4f68270..99505b50b 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetBatches.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetBatches.groovy @@ -26,6 +26,7 @@ final class ParquetBatches implements MatrixBatchIterator { private final ParquetPredicate residual private final FilterCompat.Filter filter private final String name + private final String sourcePath private final String indexMetadata private final boolean binaryAsString private ColumnMaterializer materializer @@ -39,9 +40,11 @@ final class ParquetBatches implements MatrixBatchIterator { /** Opens an input and parses its footer exactly once. */ @SuppressWarnings('AbcMetric') // Validates and snapshots all footer-dependent state before data I/O. - ParquetBatches(InputFile input, ParquetReadOptions options, String fallbackName, ParquetMetadata snapshot = null) { + ParquetBatches(InputFile input, ParquetReadOptions options, String fallbackName, ParquetMetadata snapshot = null, String sourcePath = null) { options.validate() this.options = options + this.sourcePath = sourcePath + if (options.sourceFileColumn != null && sourcePath == null) { throw new IllegalArgumentException('sourceFileColumn requires a local file') } def conf = new PlainParquetConfiguration() def baseOptions = org.apache.parquet.ParquetReadOptions.builder(conf).build() ParquetMetadata footer = snapshot @@ -57,9 +60,7 @@ final class ParquetBatches implements MatrixBatchIterator { indexMetadata = metadata[MatrixParquetReader.METADATA_INDEX_COLUMNS] String typeString = metadata[MatrixParquetReader.METADATA_COLUMN_TYPES] boolean legacy = typeString != null && !metadata.containsKey('matrix.parquet.version') - if (legacy && typeString.split(COMMA, KEEP_EMPTY).contains('[B')) { - throw new IllegalArgumentException('Unrecoverable legacy Matrix byte[]: original binary payload was not preserved') - } + binaryAsString = options.binaryAsString != null ? options.binaryAsString : legacy outputNames = options.columns == null ? full.fields*.name : new ArrayList<>(options.columns) outputNames.each { String column -> @@ -68,6 +69,8 @@ final class ParquetBatches implements MatrixBatchIterator { residual = options.filter?.bind(full, options.zoneId) Set needed = new LinkedHashSet<>(outputNames) if (options.filter != null) { needed.addAll(options.filter.columns()) } + validateLegacyBinary(full, typeString, needed, legacy) + if (options.sourceFileColumn != null && full.containsField(options.sourceFileColumn)) { throw new IllegalArgumentException('sourceFileColumn collides with file schema') } requested = new MessageType(full.name, full.fields.findAll { needed.contains(it.name) }) List inferred = MatrixParquetReader.extractFieldTypes(full) List declared = typeString == null ? [] : typeString.split(COMMA, KEEP_EMPTY).toList() @@ -101,6 +104,13 @@ final class ParquetBatches implements MatrixBatchIterator { remaining = options.limit == null ? Long.MAX_VALUE : options.limit } + private static void validateLegacyBinary(MessageType full, String typeString, Set needed, boolean legacy) { + List legacyTypes = typeString == null ? [] : typeString.split(COMMA, KEEP_EMPTY).toList() + if (legacy && legacyTypes.size() == full.fieldCount && full.fields.any { field -> needed.contains(field.name) && legacyTypes[full.getFieldIndex(field.name)] == '[B' }) { + throw new IllegalArgumentException('Unrecoverable legacy Matrix byte[]: original binary payload was not preserved') + } + } + private static boolean compatibleMetadata(Class expected, Type field) { if (expected == null) { return false } if (!field.primitive) { @@ -114,7 +124,7 @@ final class ParquetBatches implements MatrixBatchIterator { if (logical instanceof org.apache.parquet.schema.LogicalTypeAnnotation.TimeLogicalTypeAnnotation) { return expected in [java.sql.Time, java.time.LocalTime, Date] } if (logical instanceof org.apache.parquet.schema.LogicalTypeAnnotation.DateLogicalTypeAnnotation) { return expected in [java.sql.Date, java.time.LocalDate, Date] } if (logical instanceof org.apache.parquet.schema.LogicalTypeAnnotation.UUIDLogicalTypeAnnotation) { return expected == UUID } - if (logical instanceof org.apache.parquet.schema.LogicalTypeAnnotation.StringLogicalTypeAnnotation) { return expected == String } + if (MatrixParquetReader.isTextLogicalType(logical)) { return expected == String } compatiblePhysicalMetadata(expected, primitive.primitiveTypeName) } @@ -144,6 +154,8 @@ final class ParquetBatches implements MatrixBatchIterator { Matrix result = Matrix.builder(name).columnNames(outputNames).types(indexes.collect { types[it] }) .columns(copied).build() MatrixParquetReader.restoreIndex(result, indexMetadata) + if (options.sourceFileColumn != null) { result.addColumn(options.sourceFileColumn, String, [sourcePath] * result.rowCount()) } + result } @Override diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetReadOptions.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetReadOptions.groovy index 02c6a5e76..222ac9cd9 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetReadOptions.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetReadOptions.groovy @@ -141,6 +141,7 @@ class ParquetReadOptions { Set unknown = new HashSet<>(normalized.keySet()) unknown.removeAll(known) if (!unknown.empty) { throw new IllegalArgumentException("Unknown Parquet options: $unknown") } + normalized = normalized.findAll { String key, Object value -> value != null } if (normalized.containsKey('matrixname')) { String matrixName = OptionMaps.stringValueOrNull(normalized.matrixname) if (matrixName != null) { diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetWriteOptions.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetWriteOptions.groovy index e1c9dec3a..8aec792f2 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetWriteOptions.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/ParquetWriteOptions.groovy @@ -37,37 +37,37 @@ class ParquetWriteOptions { boolean dictionaryEncoding = true - /** Sets dictionaryEncoding (default true); target sizes are not hard memory limits. */ + /** Enable dictionary encoding with plain-page fallback (default true). */ ParquetWriteOptions dictionaryEncoding(boolean value) { this.dictionaryEncoding = value; this } String writerVersion = KEY_V1 - /** Sets writerVersion (default KEY_V1); target sizes are not hard memory limits. */ + /** Choose V1 or V2 value encodings and page layout (default V1). */ ParquetWriteOptions writerVersion(String value) { this.writerVersion = value; this } int columnIndexTruncateLength = 64 - /** Sets columnIndexTruncateLength (default 64); target sizes are not hard memory limits. */ + /** Maximum binary bound bytes in page indexes (default 64). */ ParquetWriteOptions columnIndexTruncateLength(int value) { this.columnIndexTruncateLength = value; this } int statisticsTruncateLength = Integer.MAX_VALUE - /** Sets statisticsTruncateLength (default Integer.MAX_VALUE); target sizes are not hard memory limits. */ + /** Maximum binary bound bytes in row-group statistics (default 2147483647). */ ParquetWriteOptions statisticsTruncateLength(int value) { this.statisticsTruncateLength = value; this } Map bloomFilterEnabled = [:] - /** Sets bloomFilterEnabled (default [:]); target sizes are not hard memory limits. */ + /** Per-primitive-column bloom overrides (default empty, disabled). */ ParquetWriteOptions bloomFilterEnabled(Map value) { this.bloomFilterEnabled = value; this } Map bloomFilterNDV = [:] - /** Sets bloomFilterNDV (default [:]); target sizes are not hard memory limits. */ + /** Positive expected distinct counts for bloom sizing (default empty). */ ParquetWriteOptions bloomFilterNDV(Map value) { this.bloomFilterNDV = value; this } boolean sizeStatisticsEnabled = true - /** Sets sizeStatisticsEnabled (default true); target sizes are not hard memory limits. */ + /** Collect column size statistics (default true). */ ParquetWriteOptions sizeStatisticsEnabled(boolean value) { this.sizeStatisticsEnabled = value; this } boolean inferPrecisionAndScale = true @@ -175,6 +175,8 @@ class ParquetWriteOptions { throw new IllegalArgumentException('Writer sizes/truncate lengths must be positive') } if (!(writerVersion in [KEY_V1, 'V2'])) { throw new IllegalArgumentException('writerVersion must be V1 or V2') } + if (bloomFilterEnabled == null || bloomFilterNDV == null) { throw new IllegalArgumentException('Bloom option maps cannot be null') } + if (bloomFilterEnabled.values().any { it == null }) { throw new IllegalArgumentException('Bloom overrides must be non-null Booleans') } if (bloomFilterNDV.values().any { it == null || it <= 0 }) { throw new IllegalArgumentException('Bloom NDVs must be positive') } if ((precision == null) != (scale == null)) { @@ -226,6 +228,7 @@ class ParquetWriteOptions { Set unknown = new HashSet<>(normalized.keySet()) unknown.removeAll(known) if (!unknown.empty) { throw new IllegalArgumentException("Unknown Parquet options: $unknown") } + normalized = normalized.findAll { String key, Object value -> value != null } if (normalized.containsKey('inferprecisionandscale')) { Object inferPrecisionAndScale = normalized.inferprecisionandscale if (inferPrecisionAndScale != null) { diff --git a/matrix-parquet/src/test/groovy/HubUrlTest.groovy b/matrix-parquet/src/test/groovy/HubUrlTest.groovy index fe1e1970d..4f67ff431 100644 --- a/matrix-parquet/src/test/groovy/HubUrlTest.groovy +++ b/matrix-parquet/src/test/groovy/HubUrlTest.groovy @@ -17,4 +17,14 @@ class HubUrlTest { assertThrows(IllegalArgumentException) { HubUrl.resolve(value) } } } + + @Test + void rejectDotSegments() { + ['.', '..', 'nested/../file', 'nested/./file'].each { path -> + assertThrows(IllegalArgumentException) { HubUrl.resolve("hf://datasets/owner/repo/$path") } + } + ['..', '%2E%2E', 'feature%2F..%2Fbranch'].each { revision -> + assertThrows(IllegalArgumentException) { HubUrl.resolve("hf://datasets/owner/repo@$revision/file") } + } + } } diff --git a/matrix-parquet/src/test/groovy/ParquetDatasetAndHttpTest.groovy b/matrix-parquet/src/test/groovy/ParquetDatasetAndHttpTest.groovy index 92cd1c2af..1ac31bd7c 100644 --- a/matrix-parquet/src/test/groovy/ParquetDatasetAndHttpTest.groovy +++ b/matrix-parquet/src/test/groovy/ParquetDatasetAndHttpTest.groovy @@ -52,6 +52,56 @@ class ParquetDatasetAndHttpTest { } } + @Test + void bulkRangesUseOneRequestAndReuseRedirectAndCache() { + byte[] bytes = new byte[2 * 1024 * 1024] + Arrays.fill(bytes, (byte) 37) + AtomicLong redirects = new AtomicLong() + AtomicLong ranges = new AtomicLong() + def server = HttpServer.create(new InetSocketAddress('127.0.0.1', 0), 0) + server.createContext('/resolve') { exchange -> + redirects.incrementAndGet() + exchange.responseHeaders.add('Location', '/object') + exchange.sendResponseHeaders(302, -1) + exchange.close() + } + server.createContext('/object') { exchange -> + ranges.incrementAndGet() + def match = exchange.requestHeaders.getFirst('Range') =~ /bytes=(\d+)-(\d+)/ + assertTrue(match.matches()) + int start = Integer.parseInt(match.group(1)) + int end = Integer.parseInt(match.group(2)) + exchange.responseHeaders.add('ETag', '"stable"') + exchange.responseHeaders.add('Content-Range', "bytes $start-$end/${bytes.length}") + exchange.sendResponseHeaders(206, end - start + 1) + exchange.responseBody.write(bytes, start, end - start + 1) + exchange.close() + } + server.start() + try { + URL url = new URI("http://127.0.0.1:${server.address.port}/resolve").toURL() + def input = new HttpRangeInputFile(url, new ParquetReadOptions()) + input.newStream().withCloseable { stream -> + def target = java.nio.ByteBuffer.allocate(1024 * 1024) + stream.readFully(target) + assertEquals(37, target.get(0)) + assertEquals(2L, ranges.get()) // Probe plus one 1 MiB bulk request. + stream.seek(0) + stream.readFully(new byte[1024 * 1024]) + assertEquals(2L, ranges.get()) // Default cache retains the bulk range. + } + assertEquals(1L, redirects.get()) + def tinyCache = new HttpRangeInputFile(url, new ParquetReadOptions(rangeCacheBytes: 1024)) + tinyCache.newStream().withCloseable { stream -> + long before = ranges.get() + stream.readFully(new byte[1024 * 1024]) + assertEquals(before + 1, ranges.get()) // Request size is independent of cache size. + } + println "HTTP bulk 1 MiB: one data request; redirects=${redirects.get()}, ranges=${ranges.get()}" + assertEquals(2L, redirects.get()) // One resolution per input, never per bulk request. + } finally { server.stop(0) } + } + @Test void rangeReadsFallbackAndObjectChange() { File file = new File(directory, 'large.parquet') diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/HttpTransportTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/HttpTransportTest.groovy index f310571ac..d8e3b54d1 100644 --- a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/HttpTransportTest.groovy +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/HttpTransportTest.groovy @@ -60,7 +60,31 @@ class HttpTransportTest { }) def input = new HttpRangeInputFile(source, new ParquetReadOptions(rangeCacheBytes: 4)) assertThrows(EOFException) { input.newStream().withCloseable { it.readFully(new byte[4]) } } - assertTrue(requests.every { it.disconnected }) + assertTrue(requests.last().disconnected) + assertFalse(requests.first().disconnected) // Fully consumed success allows keep-alive reuse. + } + + @Test + void staticUrlOverloadsUseBoundedTransportAndHubParsing() { + java.time.ZoneId zone = java.time.ZoneId.of('UTC') + [ { URL url -> MatrixParquetReader.read(url) }, + { URL url -> MatrixParquetReader.read(url, zone) }, + { URL url -> MatrixParquetReader.read(url, 'named', zone) } ].each { read -> + StubConnection request + URL url = new URL(null, 'https://example.test/file', new URLStreamHandler() { + @Override protected URLConnection openConnection(URL source) { + request = new StubConnection(source, 302, [Location: 'http://example.test/file']) + request + } + }) + assertThrows(IOException) { read(url) } + assertEquals(10000, request.connectTimeout) + assertEquals(60000, request.readTimeout) + assertTrue(request.disconnected) + } + String invalid = 'hf://datasets/owner/repo/../file.parquet' + assertThrows(IllegalArgumentException) { MatrixParquetReader.readUrl(invalid, zone) } + assertThrows(IllegalArgumentException) { MatrixParquetReader.readUrl(invalid, 'named', zone) } } @Test diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReviewRegressionTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReviewRegressionTest.groovy new file mode 100644 index 000000000..717fa6710 --- /dev/null +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReviewRegressionTest.groovy @@ -0,0 +1,162 @@ +package se.alipsa.matrix.parquet + +import static org.junit.jupiter.api.Assertions.* + +import org.apache.parquet.conf.PlainParquetConfiguration +import org.apache.parquet.example.data.simple.SimpleGroupFactory +import org.apache.parquet.hadoop.example.ExampleParquetWriter +import org.apache.parquet.io.LocalOutputFile +import org.apache.parquet.schema.MessageTypeParser +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +import se.alipsa.matrix.core.Matrix + +import java.time.LocalDateTime +import java.time.ZoneId + +class ParquetReviewRegressionTest { + @TempDir File directory + + @Test + void arrowEmptyDictionaryAndJsonAreReadable() { + File root = new File('src/test/resources/optimized') + def allNull = MatrixParquetReader.read(new File(root, 'arrow-all-null-string.parquet')) + assertEquals(String, allNull.type('s')) + assertEquals([null, null, null], allNull.column('s')) + [null, true, false].each { override -> + def json = MatrixParquetReader.builder().binaryAsString(override).read(new File(root, 'arrow-json.parquet')) + assertEquals(String, json.type('json')) + assertEquals(['{"label":"α"}', null, '{}'], json.column('json')) + } + } + + @Test + void enumIsTextAndBsonRemainsBinary() { + def schema = MessageTypeParser.parseMessageType('message external { optional binary label (ENUM); optional binary document (BSON); }') + File file = new File(directory, 'annotations.parquet') + ExampleParquetWriter.builder(new LocalOutputFile(file.toPath())).withConf(new PlainParquetConfiguration()) + .withType(schema).build().withCloseable { writer -> + writer.write(new SimpleGroupFactory(schema).newGroup().append('label', 'CATEGORY').append('document', 'binary')) + } + def result = MatrixParquetReader.builder().binaryAsString(true).read(file) + assertEquals(String, result.type('label')) + assertEquals('CATEGORY', result[0, 'label']) + assertEquals(byte[], result.type('document')) + assertArrayEquals('binary'.getBytes('UTF-8'), (byte[]) result[0, 'document']) + } + + @Test + void failedDecimalWritesPreserveExistingFileAndCleanTemporaryFiles() { + File file = new File(directory, 'safe.parquet') + MatrixParquetWriter.write(Matrix.builder().columns(d: [1.25G]).build(), file) + boolean posix = java.nio.file.Files.getFileStore(file.toPath()).supportsFileAttributeView('posix') + def permissions = java.nio.file.attribute.PosixFilePermissions.fromString('rw-r-----') + if (posix) { java.nio.file.Files.setPosixFilePermissions(file.toPath(), permissions) } + byte[] original = file.bytes + [2.255G, 123456.00G].each { value -> + assertThrows(IllegalArgumentException) { + MatrixParquetWriter.builder(Matrix.builder().columns(d: [1.00G, value]).build()) + .decimalMeta([d: [5, 2] as int[]]).write(file) + } + assertArrayEquals(original, file.bytes) + assertEquals(['safe.parquet'], directory.list().toList()) + } + MatrixParquetWriter.write(Matrix.builder().columns(d: [2.50G]).build(), file) + assertEquals(2.50G, MatrixParquetReader.read(file)[0, 'd']) + if (posix) { assertEquals(permissions, java.nio.file.Files.getPosixFilePermissions(file.toPath())) } + assertEquals(['safe.parquet'], directory.list().toList()) + } + + @Test + void nullMapOptionsPreserveDefaultsAndRejectNullBloomEntries() { + def read = ParquetReadOptions.fromMap([useColumnIndexFilter: null, binaryAsString: null]) + assertTrue(read.useColumnIndexFilter) + assertNull(read.binaryAsString) + def write = ParquetWriteOptions.fromMap([dictionaryEncoding: null, sizeStatisticsEnabled: null, + bloomFilterEnabled: null, bloomFilterNDV: null, writerVersion: null, pageSize: null]) + assertTrue(write.dictionaryEncoding) + assertTrue(write.sizeStatisticsEnabled) + assertEquals([:], write.bloomFilterEnabled) + assertEquals('V1', write.writerVersion) + assertEquals(1048576, write.pageSize) + assertThrows(IllegalArgumentException) { ParquetWriteOptions.fromMap([bloomFilterEnabled: [id: null]]) } + } + + @Test + void localSourcesAndCompatibleIndexesAgreeAcrossEntryPoints() { + File file = new File(directory, 'indexed.parquet') + def data = Matrix.builder().columns(id: [1, 2], when: [LocalDateTime.of(2024, 1, 1, 0, 0)] * 2).build() + data.createIndex('id') + MatrixParquetWriter.builder(data).zoneId('UTC').write(file) + ZoneId zone = ZoneId.of('Europe/Stockholm') + [MatrixParquetReader.read(file, zone), MatrixParquetReader.read(directory, zone), + MatrixParquetReader.read(directory, 'named', zone), MatrixParquetReader.read(directory.toPath(), zone), + MatrixParquetReader.read(directory, 'named')].each { result -> + assertTrue(result.hasIndex()) + assertEquals(['id'], result.indexedColumns()) + } + assertEquals(LocalDateTime.of(2024, 1, 1, 1, 0), MatrixParquetReader.read(directory, zone)[0, 'when']) + [file, directory].each { source -> + def result = MatrixParquetReader.builder().sourceFileColumn('_source').read(source) + assertEquals([file.canonicalPath] * 2, result.column('_source')) + assertTrue(result.hasIndex()) + MatrixParquetReader.builder().sourceFileColumn('_source').batches(source).withCloseable { batches -> + assertEquals([file.canonicalPath] * 2, batches.next().column('_source')) + } + } + MatrixParquetReader.builder().limit(1).batches([file]).withCloseable { batches -> + def limited = batches.next() + assertEquals(1, limited.rowCount()) + assertTrue(limited.hasIndex()) + } + assertThrows(IllegalArgumentException) { MatrixParquetReader.builder().sourceFileColumn('id').read(file) } + } + + @Test + void shardBatchesAndCollectedReadsUseTheCommonIndexPolicy() { + List files = [] + ['id', 'text'].eachWithIndex { indexColumn, int number -> + File file = new File(directory, "shard-${number}.parquet") + def matrix = Matrix.builder().columns(id: [1, 2], text: ['a', 'b']).build() + matrix.createIndex(indexColumn) + MatrixParquetWriter.write(matrix, file) + files.add(file) + } + assertFalse(MatrixParquetReader.read(files).hasIndex()) + MatrixParquetReader.builder().batches(files).withCloseable { batches -> + while (batches.hasNext()) { assertFalse(batches.next().hasIndex()) } + } + assertFalse(MatrixParquetReader.builder().columns(['text']).read(files[0]).hasIndex()) + } + + @Test + void workerThreadsAreDaemonAndNestedErrorsIncludeTheirPath() { + File file = new File(directory, 'workers.parquet') + MatrixParquetWriter.write(Matrix.builder().columns(id: [1, 2]).build(), file) + MatrixParquetReader.builder().batches([file]).withCloseable { batches -> + assertTrue(batches.hasNext()) + def workers = Thread.allStackTraces.keySet().findAll { it.name == 'matrix-parquet-dataset' } + assertFalse(workers.empty) + assertTrue(workers.every { it.daemon }) + } + def invalid = Matrix.builder().columns(items: [[[value: [1, 'mixed']]]]).build() + def failure = assertThrows(IllegalArgumentException) { MatrixParquetWriter.writeBytes(invalid) } + assertTrue(failure.message.contains('items.element.value.element'), failure.message) + } + + @Test + void projectedLegacyBinaryFailsOnlyWhenRequired() { + def schema = MessageTypeParser.parseMessageType('message legacy { optional int32 id; optional binary payload; }') + File file = new File(directory, 'legacy.parquet') + ExampleParquetWriter.builder(new LocalOutputFile(file.toPath())).withConf(new PlainParquetConfiguration()) + .withType(schema).withExtraMetaData(['matrix.columnTypes': 'java.lang.Integer,[B']).build().withCloseable { writer -> + writer.write(new SimpleGroupFactory(schema).newGroup().append('id', 1).append('payload', '[B@lost')) + } + assertEquals([1], MatrixParquetReader.builder().columns(['id']).read(file).column('id')) + assertThrows(IllegalArgumentException) { MatrixParquetReader.read(file) } + assertThrows(IllegalArgumentException) { + MatrixParquetReader.builder().columns(['id']).filter(Predicates.isNull('payload')).read(file) + } + } +} diff --git a/matrix-parquet/src/test/python/generate_optimized.py b/matrix-parquet/src/test/python/generate_optimized.py index 6d1eb435a..39c455a35 100644 --- a/matrix-parquet/src/test/python/generate_optimized.py +++ b/matrix-parquet/src/test/python/generate_optimized.py @@ -42,6 +42,14 @@ assert pq.read_table(out / name).equals(table) assert (out / name).stat().st_size <= 1024 * 1024 manifest['files'][name] = {'options': options, 'sha256': hashlib.sha256((out / name).read_bytes()).hexdigest(), 'schema': str(table.schema)} +# Arrow emits an empty dictionary for an all-null string column. +for name, extra in { + 'arrow-all-null-string.parquet': pa.table({'s': pa.array([None] * 3, type=pa.string())}), + 'arrow-json.parquet': pa.table({'json': pa.array(['{"label":"α"}', None, '{}'], type=pa.json_())}), +}.items(): + pq.write_table(extra, out / name, use_dictionary=True, write_page_index=True) + manifest['files'][name] = {'sha256': hashlib.sha256((out / name).read_bytes()).hexdigest(), + 'rows': extra.num_rows, 'schema': str(extra.schema)} # A synthetic legacy [B footer exercises contextual rejection; the original writer cannot # produce a valid byte[] file in this checkout (InvalidSchemaException, captured separately). legacy_binary = pa.table({'payload': pa.array([b'[B@legacy-text'], type=pa.binary())}) diff --git a/matrix-parquet/src/test/resources/optimized/README.md b/matrix-parquet/src/test/resources/optimized/README.md index 3ec439776..65bfc6ae8 100644 --- a/matrix-parquet/src/test/resources/optimized/README.md +++ b/matrix-parquet/src/test/resources/optimized/README.md @@ -40,3 +40,6 @@ regex accepts dots/dashes (`[\w.-]+`), and revision separation occurs immediatel namespaced repository, so deep-path `@` is literal. Special convert/pr refs remain unescaped; other revision slashes are encoded. Revision text is unquoted once; paths are literal and quoted, including existing percent signs. The Java parser follows the tested 2.2.0 behavior. + +The additional three-row Arrow fixtures cover an all-null STRING dictionary and JSON logical +annotations. Their file-level manifest rows override the main 4096-row dataset count. diff --git a/matrix-parquet/src/test/resources/optimized/arrow-all-null-string.parquet b/matrix-parquet/src/test/resources/optimized/arrow-all-null-string.parquet new file mode 100644 index 0000000000000000000000000000000000000000..58ebf06a7c1b4bdaf1cb2aa0f75e8ec4aa6c2605 GIT binary patch literal 418 zcmXw0!Ab)$5S?90?4ih7n2<;gy)+=SwN>juyv&vs#VxiAB6usMtRA%Mu3GTqr}(MP z?Ak{%^YUggGb9U2N7z8XFku620JeCD?H($Ocq@Pd6SB<29tG9_F9fUAi5JOfj~AK= zZ*17!7>p{{&zsj3hJdYZXo9i}iQ@`dS!bOGhb=n>6b-BeO9i%9Pa{)A zT;W=SRaQEI{t56{KcNS{pnOxO+}*S&P46GC>Xzm=I2D|eU|CtBJE9%M8O?|?nq-w4 z$3$~s6QW!3eBOILO+|VT!;F0<_iv;QV=2isrn4%OgN%y)>+Irh;hf6)eiR}woqSci me{Med>pbtTiekI#d~J&IW&74S8x4kop>^_F^8jGMzw-xJW=?_t literal 0 HcmV?d00001 diff --git a/matrix-parquet/src/test/resources/optimized/arrow-json.parquet b/matrix-parquet/src/test/resources/optimized/arrow-json.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c00a860478f46cadd0827c6db4a91b5ffbb6674a GIT binary patch literal 676 zcmZ8f+fLg+5FIb6MWqN)q+NNDAFQZYtrTcT0|HbY)_~nhT7o0e5cMT?t#GjuHwjQc zNPX;Mf2IFX^|9*bbY@*t(Ia`sb31cJnwAPzd5PC}qlpF<0IUyl0B}=D`%#iswkkir zS8f?aZaMG;E~?z*^Z1zn4;@O(JTPXMe8i+RcfU(o44aq~?5WF2AZ^j*8v;NI2Y5+V z*T6C-@o|#%IRjo^Vpi#NG8!(q%^yfBOleEo1)G=U8kqC4$Z{r3@x>5d4Y%vg+Ux8I z4*7pF$SQ3^w18q6ybz8wC8+@M%ixRRo56h`9@CvgE)-{#RT%ES-zpSYzu7a5fkGV6 zgLtFVrSb;KS8AYkbV_-iQg11_qtq+1=-*lN_9%2j528)YSQjW2t6hpRjh( z@-j@L;kz6;*Q3mz_KI<+ANSOIXg(c9{~iMJ_WbtY*=Bb?>oh(Fo%6V!UPPz)wAT%f zdx2j&D)>;}0r4TWx50Aq{`k{*GF^`Ie7Qd!k1m~yWIQ<;4W0VxN^PYkZ5uD(F\n child 0, bytes: binary\n child 1, path: string\ntags: list\n child 0, item: string" }, + "arrow-all-null-string.parquet": { + "sha256": "12949788c7fc627d17a79937f560e82b4b4b6d7977b220c32bc74e1ceb668e71", + "rows": 3, + "schema": "s: string" + }, + "arrow-json.parquet": { + "sha256": "b017865f5ec5157bae2147e206f0d304d8c24f031968545df3f7e76208397667", + "rows": 3, + "schema": "json: extension" + }, "legacy-binary-metadata.parquet": { "provenance": "synthetic legacy [B metadata, not claimed as original recoverable bytes", "sha256": "f46e7f0cc774068aba6cc44575209f92d8efb01ef2c008a403777c532943e8d1", From d6caded3fbc42e520e34ac68190ad0e8b9c72eb0 Mon Sep 17 00:00:00 2001 From: Per Nyfelt Date: Sat, 17 Oct 2026 20:45:31 +0200 Subject: [PATCH 3/5] Preserve Parquet output permissions and symbolic links --- matrix-parquet/readme.md | 4 ++- matrix-parquet/release.md | 6 +++- .../req/optimized-parquet-review.md | 29 ++++++++++++++++ .../matrix/parquet/HttpRangeInputFile.groovy | 6 ++++ .../matrix/parquet/MatrixParquetWriter.groovy | 4 ++- .../groovy/ParquetDatasetAndHttpTest.groovy | 19 +++++++++++ .../ParquetReviewRegressionTest.groovy | 33 +++++++++++++++++++ 7 files changed, 98 insertions(+), 3 deletions(-) diff --git a/matrix-parquet/readme.md b/matrix-parquet/readme.md index f94298946..77975140b 100644 --- a/matrix-parquet/readme.md +++ b/matrix-parquet/readme.md @@ -178,7 +178,9 @@ not enable a filter by themselves. Column settings use physical primitive paths Files are written to a temporary sibling and atomically replace the destination after successful close. Failed writes preserve an existing destination; filesystems without atomic replacement -support fail safely. Existing POSIX permissions are retained. +support fail safely with `AtomicMoveNotSupportedException`. New files use normal process/umask +permissions; existing POSIX permissions are retained. Existing symbolic links are resolved before +writing, so the link is preserved and its target is replaced. JSON and ENUM annotations decode as UTF-8 String; BSON remains binary. Strings are STRING-annotated recursively; byte[] stays raw BINARY. Byte/Short use signed width diff --git a/matrix-parquet/release.md b/matrix-parquet/release.md index c707f0572..25e394e0c 100644 --- a/matrix-parquet/release.md +++ b/matrix-parquet/release.md @@ -52,7 +52,11 @@ metadata, and reject empty selections and duplicate paths. Review fixes also cover empty Arrow dictionaries; consistent static URL/Hub and directory overloads; local source columns; daemon dataset workers; null option-map defaults; path-segment validation; projection around legacy `[B` columns; one validated type-policy scan per column; and atomic output -replacement on success. HTTP bulk requests reuse resolved targets and a separately bounded cache. +replacement on success. New files use normal process/umask permissions; existing POSIX permissions +are retained, and writes through existing symbolic links update the resolved target while preserving +the link. Local file writes require same-directory atomic replacement support; unsupported network +or FUSE filesystems raise `AtomicMoveNotSupportedException` and leave existing output unchanged. +HTTP bulk requests reuse resolved targets and a separately bounded cache. Native CDC writing remains deferred to upstream parquet-java #3818; the Python rewrite preserves custom metadata. Hadoop stays as a runtime dependency because codec initialization still loads it. diff --git a/matrix-parquet/req/optimized-parquet-review.md b/matrix-parquet/req/optimized-parquet-review.md index d8f002b49..4eb199396 100644 --- a/matrix-parquet/req/optimized-parquet-review.md +++ b/matrix-parquet/req/optimized-parquet-review.md @@ -69,3 +69,32 @@ still permits the same single 1 MiB bulk request. The test checks byte-array and readFully paths. Successful bodies remain eligible for JDK keep-alive pooling; socket-level connection reuse and live Hub latency were not measured. Existing selective-transfer, page-index and bloom-filter byte-reduction tests also pass. + +## 10. Follow-up output and range-buffer review + +- 10.1 [x] Create temporary siblings with ordinary file creation permissions, so new output + follows the process umask. Compare new Parquet permissions with an ordinary sibling file on + POSIX, while retaining the existing-destination permission regression. +- 10.2 [x] Resolve existing output paths before creating the sibling and replacing the target. + Test that relative symbolic links survive both failed and successful writes, and that the + resolved file retains its contents on failure and receives the new rows on success. +- 10.3 [x] Document the same-directory atomic replacement requirement and + `AtomicMoveNotSupportedException` in the release notes and module guide. Keep safe failure + rather than introduce a replacement fallback with weaker failure guarantees. +- 10.4 [x] Read directly into array-backed ByteBuffers using their array offset and current + position. Retain the bounded scratch buffer for direct buffers. Exercise sliced buffers with + nonzero positions, sentinel bytes outside the slice, direct buffers, EOF and empty reads in + the existing HTTP request-count regression. + +Follow-up verification passed, in the required order: + +```bash +./gradlew :matrix-parquet:codenarcMain :matrix-parquet:codenarcTest :matrix-parquet:spotlessCheck --console=plain +PARQUET_PYTHON=/tmp/matrix-parquet-fixtures/bin/python \ + ./gradlew :matrix-parquet:test :matrix-parquet:compileConsumerApi -PrunSlowTests=true --console=plain +./gradlew test -Pheadless=true --console=plain +``` + +The Parquet suite reported **135 tests, 130 passed, zero failed, five intentional skips**, +including successful POSIX permission/symlink tests, Python interoperability and consumer API +compilation. The full repository task passed; unaffected tasks may be up-to-date. diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpRangeInputFile.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpRangeInputFile.groovy index c93341cf0..a425f2466 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpRangeInputFile.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/HttpRangeInputFile.groovy @@ -142,6 +142,12 @@ final class HttpRangeInputFile implements InputFile { } @Override int read(ByteBuffer target) { + if (target.isReadOnly()) { throw new java.nio.ReadOnlyBufferException() } + if (target.hasArray()) { + int result = read(target.array(), target.arrayOffset() + target.position(), target.remaining()) + if (result > 0) { target.position(target.position() + result) } + return result + } byte[] bytes = new byte[Math.min(target.remaining(), MAX_REQUEST_BYTES)] int result = read(bytes, 0, bytes.length) if (result > 0) { target.put(bytes, 0, result) } diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy index 53afd7c13..d8a5e1599 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy @@ -566,7 +566,9 @@ class MatrixParquetWriter { CompressionCodecName compressionCodec = CompressionCodecName.SNAPPY, RoundingMode roundingMode = RoundingMode.UNNECESSARY, ParquetWriteOptions options = new ParquetWriteOptions()) { MessageType schema = resolved.schema java.nio.file.Path target = file.toPath().toAbsolutePath() - java.nio.file.Path temporary = java.nio.file.Files.createTempFile(target.parent, '.matrix-parquet-', '.tmp') + if (java.nio.file.Files.exists(target)) { target = target.toRealPath() } + java.nio.file.Path temporary = target.resolveSibling(".matrix-parquet-${UUID.randomUUID()}.tmp") + java.nio.file.Files.createFile(temporary) try { if (java.nio.file.Files.exists(target) && java.nio.file.Files.getFileStore(target).supportsFileAttributeView('posix')) { java.nio.file.Files.setPosixFilePermissions(temporary, java.nio.file.Files.getPosixFilePermissions(target)) diff --git a/matrix-parquet/src/test/groovy/ParquetDatasetAndHttpTest.groovy b/matrix-parquet/src/test/groovy/ParquetDatasetAndHttpTest.groovy index 1ac31bd7c..257de97a9 100644 --- a/matrix-parquet/src/test/groovy/ParquetDatasetAndHttpTest.groovy +++ b/matrix-parquet/src/test/groovy/ParquetDatasetAndHttpTest.groovy @@ -89,6 +89,25 @@ class ParquetDatasetAndHttpTest { stream.seek(0) stream.readFully(new byte[1024 * 1024]) assertEquals(2L, ranges.get()) // Default cache retains the bulk range. + def backing = java.nio.ByteBuffer.allocate(1024 * 1024 + 32) + backing.position(13) + backing.limit(1024 * 1024 + 13) + def slice = backing.slice() + slice.position(7) + stream.seek(0) + stream.readFully(slice) + assertEquals(slice.limit(), slice.position()) + assertEquals(0, backing.get(19)) + assertEquals(37, backing.get(20)) + assertEquals(0, backing.array()[backing.limit()]) + assertEquals(2L, ranges.get()) + def direct = java.nio.ByteBuffer.allocateDirect(1024) + stream.seek(0) + stream.readFully(direct) + assertEquals(37, direct.get(0)) + stream.seek(bytes.length) + assertEquals(-1, stream.read(java.nio.ByteBuffer.allocate(1))) + assertEquals(0, stream.read(java.nio.ByteBuffer.allocate(0))) } assertEquals(1L, redirects.get()) def tinyCache = new HttpRangeInputFile(url, new ParquetReadOptions(rangeCacheBytes: 1024)) diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReviewRegressionTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReviewRegressionTest.groovy index 717fa6710..040296883 100644 --- a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReviewRegressionTest.groovy +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReviewRegressionTest.groovy @@ -68,6 +68,39 @@ class ParquetReviewRegressionTest { assertEquals(['safe.parquet'], directory.list().toList()) } + @Test + void newFilePermissionsMatchOrdinaryFiles() { + org.junit.jupiter.api.Assumptions.assumeTrue( + java.nio.file.Files.getFileStore(directory.toPath()).supportsFileAttributeView('posix')) + def ordinary = java.nio.file.Files.createFile(new File(directory, 'ordinary').toPath()) + File output = new File(directory, 'new.parquet') + MatrixParquetWriter.write(Matrix.builder().columns(id: [1]).build(), output) + assertEquals(java.nio.file.Files.getPosixFilePermissions(ordinary), + java.nio.file.Files.getPosixFilePermissions(output.toPath())) + } + + @Test + void writingThroughSymbolicLinkPreservesLinkAndUpdatesRealFile() { + org.junit.jupiter.api.Assumptions.assumeTrue( + java.nio.file.Files.getFileStore(directory.toPath()).supportsFileAttributeView('posix')) + File output = new File(directory, 'real.parquet') + MatrixParquetWriter.write(Matrix.builder().columns(d: [1.25G]).build(), output) + def link = new File(directory, 'link.parquet').toPath() + java.nio.file.Files.createSymbolicLink(link, output.toPath().fileName) + byte[] original = output.bytes + assertThrows(IllegalArgumentException) { + MatrixParquetWriter.builder(Matrix.builder().columns(d: [2.255G]).build()) + .decimalMeta([d: [5, 2] as int[]]).write(link.toFile()) + } + assertTrue(java.nio.file.Files.isSymbolicLink(link)) + assertArrayEquals(original, output.bytes) + MatrixParquetWriter.write(Matrix.builder().columns(d: [2.50G]).build(), link.toFile()) + assertTrue(java.nio.file.Files.isSymbolicLink(link)) + assertEquals(output.toPath().fileName, java.nio.file.Files.readSymbolicLink(link)) + assertEquals(2.50G, MatrixParquetReader.read(output)[0, 'd']) + assertEquals(['link.parquet', 'real.parquet'], directory.list().toList().sort()) + } + @Test void nullMapOptionsPreserveDefaultsAndRejectNullBloomEntries() { def read = ParquetReadOptions.fromMap([useColumnIndexFilter: null, binaryAsString: null]) From 2be35e7897bfa900f67069ee53d3784c9733c04c Mon Sep 17 00:00:00 2001 From: Per Nyfelt Date: Sat, 17 Oct 2026 20:58:02 +0200 Subject: [PATCH 4/5] Preserve dangling symlinks when writing Parquet --- matrix-parquet/readme.md | 4 +- .../req/optimized-parquet-review.md | 22 ++++++++++ .../matrix/parquet/MatrixParquetWriter.groovy | 15 ++++++- .../ParquetReviewRegressionTest.groovy | 42 +++++++++++++++++++ 4 files changed, 80 insertions(+), 3 deletions(-) diff --git a/matrix-parquet/readme.md b/matrix-parquet/readme.md index 77975140b..9d32c5c9c 100644 --- a/matrix-parquet/readme.md +++ b/matrix-parquet/readme.md @@ -180,7 +180,9 @@ Files are written to a temporary sibling and atomically replace the destination close. Failed writes preserve an existing destination; filesystems without atomic replacement support fail safely with `AtomicMoveNotSupportedException`. New files use normal process/umask permissions; existing POSIX permissions are retained. Existing symbolic links are resolved before -writing, so the link is preserved and its target is replaced. +writing, including dangling links, so the link is preserved and its target is created or replaced. +Abrupt JVM termination can leave `.matrix-parquet-.tmp` siblings beside the resolved output; +these are unpublished temporary output and can be removed once no writer is using them. JSON and ENUM annotations decode as UTF-8 String; BSON remains binary. Strings are STRING-annotated recursively; byte[] stays raw BINARY. Byte/Short use signed width diff --git a/matrix-parquet/req/optimized-parquet-review.md b/matrix-parquet/req/optimized-parquet-review.md index 4eb199396..a0f21a03e 100644 --- a/matrix-parquet/req/optimized-parquet-review.md +++ b/matrix-parquet/req/optimized-parquet-review.md @@ -98,3 +98,25 @@ PARQUET_PYTHON=/tmp/matrix-parquet-fixtures/bin/python \ The Parquet suite reported **135 tests, 130 passed, zero failed, five intentional skips**, including successful POSIX permission/symlink tests, Python interoperability and consumer API compilation. The full repository task passed; unaffected tasks may be up-to-date. + +## 11. Dangling symlinks and interrupted-write documentation + +- 11.1 [x] Resolve symbolic links even when their destination does not exist, including relative + and absolute targets and link chains. Reject cycles before creating temporary output. Test + successful target creation and failed writes preserving dangling links and absent targets. +- 11.2 [x] Explain that abrupt JVM termination may leave `.matrix-parquet-.tmp` siblings + beside the resolved output and that unused leftovers can be removed. + +Verification passed in order: + +```bash +./gradlew :matrix-parquet:codenarcMain :matrix-parquet:codenarcTest --console=plain +./gradlew :matrix-parquet:spotlessCheck --console=plain +PARQUET_PYTHON=/tmp/matrix-parquet-fixtures/bin/python \ + ./gradlew :matrix-parquet:test :matrix-parquet:compileConsumerApi -PrunSlowTests=true --console=plain +./gradlew test -Pheadless=true --console=plain +``` + +The Parquet suite reported **137 tests, 132 passed, zero failed, five intentional skips**. +The POSIX dangling-link and cycle tests ran successfully, as did Python interoperability, +consumer API compilation and the full repository suite; unaffected tasks may be up-to-date. diff --git a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy index d8a5e1599..351b683ff 100644 --- a/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy +++ b/matrix-parquet/src/main/groovy/se/alipsa/matrix/parquet/MatrixParquetWriter.groovy @@ -565,8 +565,7 @@ class MatrixParquetWriter { private static File writeInternal(Matrix matrix, File file, WriteSchema resolved, CompressionCodecName compressionCodec = CompressionCodecName.SNAPPY, RoundingMode roundingMode = RoundingMode.UNNECESSARY, ParquetWriteOptions options = new ParquetWriteOptions()) { MessageType schema = resolved.schema - java.nio.file.Path target = file.toPath().toAbsolutePath() - if (java.nio.file.Files.exists(target)) { target = target.toRealPath() } + java.nio.file.Path target = resolveOutputTarget(file.toPath().toAbsolutePath()) java.nio.file.Path temporary = target.resolveSibling(".matrix-parquet-${UUID.randomUUID()}.tmp") java.nio.file.Files.createFile(temporary) try { @@ -586,6 +585,18 @@ class MatrixParquetWriter { } finally { java.nio.file.Files.deleteIfExists(temporary) } } + private static java.nio.file.Path resolveOutputTarget(java.nio.file.Path path) { + java.nio.file.Path target = path + Set visited = [] as Set + while (java.nio.file.Files.isSymbolicLink(target)) { + if (!visited.add(target.toAbsolutePath().normalize())) { + throw new java.nio.file.FileSystemLoopException(path.toString()) + } + target = target.resolveSibling(java.nio.file.Files.readSymbolicLink(target)) + } + java.nio.file.Files.exists(target) ? target.toRealPath() : target + } + private static final String FORMAT_VERSION = '2' private static final String DOT = '.' diff --git a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReviewRegressionTest.groovy b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReviewRegressionTest.groovy index 040296883..01d70fb25 100644 --- a/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReviewRegressionTest.groovy +++ b/matrix-parquet/src/test/groovy/se/alipsa/matrix/parquet/ParquetReviewRegressionTest.groovy @@ -101,6 +101,48 @@ class ParquetReviewRegressionTest { assertEquals(['link.parquet', 'real.parquet'], directory.list().toList().sort()) } + @Test + void danglingSymbolicLinksCreateTheirTargetsAndSurviveFailedWrites() { + org.junit.jupiter.api.Assumptions.assumeTrue( + java.nio.file.Files.getFileStore(directory.toPath()).supportsFileAttributeView('posix')) + [false, true].each { absolute -> + File output = new File(directory, "target-${absolute}.parquet") + def link = new File(directory, "dangling-${absolute}.parquet").toPath() + def destination = absolute ? output.toPath().toAbsolutePath() : output.toPath().fileName + java.nio.file.Files.createSymbolicLink(link, destination) + def chain = new File(directory, "chain-${absolute}.parquet").toPath() + java.nio.file.Files.createSymbolicLink(chain, link.fileName) + assertFalse(output.exists()) + assertThrows(IllegalArgumentException) { + MatrixParquetWriter.builder(Matrix.builder().columns(d: [2.255G]).build()) + .decimalMeta([d: [5, 2] as int[]]).write(chain.toFile()) + } + assertFalse(output.exists()) + assertTrue(java.nio.file.Files.isSymbolicLink(link)) + assertTrue(java.nio.file.Files.isSymbolicLink(chain)) + MatrixParquetWriter.write(Matrix.builder().columns(d: [2.50G]).build(), chain.toFile()) + assertEquals(destination, java.nio.file.Files.readSymbolicLink(link)) + assertEquals(link.fileName, java.nio.file.Files.readSymbolicLink(chain)) + assertEquals(2.50G, MatrixParquetReader.read(output)[0, 'd']) + } + assertFalse(directory.list().any { it.startsWith('.matrix-parquet-') }) + } + + @Test + void cyclicSymbolicLinksFailWithoutReplacingLinks() { + org.junit.jupiter.api.Assumptions.assumeTrue( + java.nio.file.Files.getFileStore(directory.toPath()).supportsFileAttributeView('posix')) + def first = new File(directory, 'first.parquet').toPath() + def second = new File(directory, 'second.parquet').toPath() + java.nio.file.Files.createSymbolicLink(first, second.fileName) + java.nio.file.Files.createSymbolicLink(second, first.fileName) + assertThrows(java.nio.file.FileSystemLoopException) { + MatrixParquetWriter.write(Matrix.builder().columns(id: [1]).build(), first.toFile()) + } + assertTrue(java.nio.file.Files.isSymbolicLink(first)) + assertTrue(java.nio.file.Files.isSymbolicLink(second)) + } + @Test void nullMapOptionsPreserveDefaultsAndRejectNullBloomEntries() { def read = ParquetReadOptions.fromMap([useColumnIndexFilter: null, binaryAsString: null]) From 68ab1377abf05ed64db67353affd8ffc26e5d461 Mon Sep 17 00:00:00 2001 From: Per Nyfelt Date: Sat, 17 Oct 2026 21:07:31 +0200 Subject: [PATCH 5/5] Align BOM with Parquet 0.8.0 snapshot --- matrix-bom/bom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix-bom/bom.xml b/matrix-bom/bom.xml index c34d6cc9a..5202867c5 100644 --- a/matrix-bom/bom.xml +++ b/matrix-bom/bom.xml @@ -46,7 +46,7 @@ 2.3.3-SNAPSHOT 0.1.1-SNAPSHOT 0.1.3-SNAPSHOT - 0.7.1-SNAPSHOT + 0.8.0-SNAPSHOT 0.7.0-SNAPSHOT 2.5.1-SNAPSHOT 2.5.1-SNAPSHOT